From 8e55e4fbc41ecdc93e382e3fb7374aaf49cb755c Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 28 Sep 2017 23:24:38 +0100 Subject: [PATCH 01/76] Implements Nullable support --- src/runtime/converter.cs | 8 +++++++- src/runtime/methodbinder.cs | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 13498e3dc..4251c80fa 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -436,6 +436,12 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return false; } + var underlyingType = Nullable.GetUnderlyingType(obType); + if (underlyingType != null) + { + return ToManagedValue(value, underlyingType, out result, setError); + } + return ToPrimitive(value, obType, out result, setError); } @@ -845,7 +851,7 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s int size = Runtime.PySequence_Size(value); result = null; - if (size < 0) + if (size < 0 || elementType.IsGenericType) { if (setError) { diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index eeec8b89d..74c069a09 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -394,8 +394,14 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth } if (!typematch) { + // this takes care of nullables + var underlyingType = Nullable.GetUnderlyingType(pi[n].ParameterType); + if (underlyingType == null) + { + underlyingType = pi[n].ParameterType; + } // this takes care of enum values - TypeCode argtypecode = Type.GetTypeCode(pi[n].ParameterType); + TypeCode argtypecode = Type.GetTypeCode(underlyingType); TypeCode paramtypecode = Type.GetTypeCode(clrtype); if (argtypecode == paramtypecode) { From c6d56bd38cf8636033a6315ad450c4d6bb987629 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 28 Sep 2017 23:28:39 +0100 Subject: [PATCH 02/76] Implements Implicit Conversion --- src/runtime/converter.cs | 14 ++++++++++++++ src/runtime/methodbinder.cs | 7 +++++++ 2 files changed, 21 insertions(+) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 4251c80fa..4f29019ba 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -442,6 +442,20 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return ToManagedValue(value, underlyingType, out result, setError); } + var opImplicit = obType.GetMethod("op_Implicit", new[] { obType }); + if (opImplicit != null) + { + if (ToManagedValue(value, opImplicit.ReturnType, out result, setError)) + { + opImplicit = obType.GetMethod("op_Implicit", new[] { result.GetType() }); + if (opImplicit != null) + { + result = opImplicit.Invoke(null, new[] { result }); + } + return opImplicit != null; + } + } + return ToPrimitive(value, obType, out result, setError); } diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 74c069a09..e956bce55 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -408,6 +408,13 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth typematch = true; clrtype = pi[n].ParameterType; } + // this takes care of implicit conversions + var opImplicit = pi[n].ParameterType.GetMethod("op_Implicit", new[] { clrtype }); + if (opImplicit != null) + { + typematch = opImplicit.ReturnType == pi[n].ParameterType; + clrtype = pi[n].ParameterType; + } } Runtime.XDecref(pyoptype); if (!typematch) From 4e4e9809d542ce1f4ad51da45227d3a9ef5a0d75 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Mon, 2 Apr 2018 11:53:48 +0100 Subject: [PATCH 03/76] Adds method name to "no method matches" error --- src/runtime/methodbinder.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index e956bce55..95fc0b026 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -186,6 +186,13 @@ internal static int GetPrecedence(MethodBase mi) val += ArgPrecedence(pi[i].ParameterType); } + var info = mi as MethodInfo; + if (info != null) + { + val += ArgPrecedence(info.ReturnType); + val += mi.DeclaringType == mi.ReflectedType ? 0 : 3000; + } + return val; } @@ -200,6 +207,11 @@ internal static int ArgPrecedence(Type t) return 3000; } + if (t.IsAssignableFrom(typeof(PyObject))) + { + return -1; + } + TypeCode tc = Type.GetTypeCode(t); // TODO: Clean up switch (tc) From 6dad4229c155d4e89a8ddd24b055d686a6faacc9 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 28 Sep 2017 23:01:37 +0100 Subject: [PATCH 04/76] Implements System.Decimal support Convertes System.Decimal to decimal.decimal and vice-versa --- src/runtime/converter.cs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 4f29019ba..7ca4aa83e 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -30,6 +30,7 @@ private Converter() private static Type flagsType; private static Type boolType; private static Type typeType; + private static IntPtr decimalCtor; static Converter() { @@ -45,6 +46,9 @@ static Converter() flagsType = typeof(FlagsAttribute); boolType = typeof(Boolean); typeType = typeof(Type); + + IntPtr decimalMod = Runtime.PyImport_ImportModule("decimal"); + if (decimalMod == null) throw new PythonException(); } @@ -100,6 +104,9 @@ internal static IntPtr GetPythonTypeByAlias(Type op) if (op == boolType) return Runtime.PyBoolType; + if (op == decimalType) + return Runtime.PyFloatType; + return IntPtr.Zero; } @@ -229,6 +236,14 @@ internal static IntPtr ToPython(object value, Type type) case TypeCode.UInt64: return Runtime.PyLong_FromUnsignedLongLong((ulong)value); + case TypeCode.Decimal: + string d2s = ((decimal)value).ToString(nfi); + IntPtr d2p = Runtime.PyString_FromString(d2s); + IntPtr decimalArgs = Runtime.PyTuple_New(1); + Runtime.PyTuple_SetItem(decimalArgs, 0, d2p); + + return Runtime.PyObject_CallObject(decimalCtor, decimalArgs); + default: if (value is IEnumerable) { @@ -821,6 +836,18 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo Runtime.XDecref(op); result = d; return true; + + case TypeCode.Decimal: + op = Runtime.PyObject_Str(value); + decimal m; + string sm = Runtime.GetManagedString(op); + if (!Decimal.TryParse(sm, NumberStyles.Number, nfi, out m)) + { + goto type_error; + } + Runtime.XDecref(op); + result = m; + return true; } From 570e0f8a92308e50ecc42d9aa9a5c1b45e0e5ddf Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Mon, 2 Apr 2018 11:51:09 +0100 Subject: [PATCH 05/76] Fixes decimal support --- src/runtime/converter.cs | 5 ++++- src/runtime/methodbinder.cs | 6 ++++++ src/runtime/runtime.cs | 9 +++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 7ca4aa83e..351a1c83a 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -75,6 +75,9 @@ internal static Type GetTypeByAlias(IntPtr op) if (op == Runtime.PyBoolType) return boolType; + if (op == Runtime.PyDecimalType) + return decimalType; + return null; } @@ -105,7 +108,7 @@ internal static IntPtr GetPythonTypeByAlias(Type op) return Runtime.PyBoolType; if (op == decimalType) - return Runtime.PyFloatType; + return Runtime.PyDecimalType; return IntPtr.Zero; } diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 95fc0b026..48dc1eb9a 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -420,6 +420,12 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth typematch = true; clrtype = pi[n].ParameterType; } + // accepts non-decimal numbers in decimal parameters + if (underlyingType == typeof(decimal)) + { + clrtype = pi[n].ParameterType; + typematch = Converter.ToManaged(op, clrtype, out arg, false); + } // this takes care of implicit conversions var opImplicit = pi[n].ParameterType.GetMethod("op_Implicit", new[] { clrtype }); if (opImplicit != null) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index b08a56622..5ef06f3cc 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -296,6 +296,14 @@ internal static void Initialize() PyFloatType = PyObject_Type(op); XDecref(op); + IntPtr decimalMod = PyImport_ImportModule("decimal"); + IntPtr decimalCtor = PyObject_GetAttrString(decimalMod, "Decimal"); + op = PyObject_CallObject(decimalCtor, IntPtr.Zero); + PyDecimalType = PyObject_Type(op); + XDecref(op); + XDecref(decimalMod); + XDecref(decimalCtor); + #if PYTHON3 PyClassType = IntPtr.Zero; PyInstanceType = IntPtr.Zero; @@ -389,6 +397,7 @@ internal static int AtExit() internal static IntPtr PyBoolType; internal static IntPtr PyNoneType; internal static IntPtr PyTypeType; + internal static IntPtr PyDecimalType; #if PYTHON3 internal static IntPtr PyBytesType; From c355ab41554723b64433fcd199dea669dad40c85 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 28 Sep 2017 23:17:41 +0100 Subject: [PATCH 06/76] Implements System.DateTime support Converts System.DateTime and System.TimeSpan to datetime.datetime and datetime.timedelta and vice-versa --- src/runtime/converter.cs | 102 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 351a1c83a..c955a880f 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -31,6 +31,9 @@ private Converter() private static Type boolType; private static Type typeType; private static IntPtr decimalCtor; + private static IntPtr dateTimeCtor; + private static IntPtr timeSpanCtor; + private static IntPtr tzInfoCtor; static Converter() { @@ -49,6 +52,34 @@ static Converter() IntPtr decimalMod = Runtime.PyImport_ImportModule("decimal"); if (decimalMod == null) throw new PythonException(); + + IntPtr dateTimeMod = Runtime.PyImport_ImportModule("datetime"); + if (dateTimeMod == null) throw new PythonException(); + + decimalCtor = Runtime.PyObject_GetAttrString(decimalMod, "Decimal"); + if (decimalCtor == null) throw new PythonException(); + + dateTimeCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "datetime"); + if (dateTimeCtor == null) throw new PythonException(); + + timeSpanCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "timedelta"); + if (timeSpanCtor == null) throw new PythonException(); + + IntPtr tzInfoMod = PythonEngine.ModuleFromString("custom_tzinfo", + "from datetime import timedelta, tzinfo\n" + + "class GMT(tzinfo):\n" + + " def __init__(self, hours, minutes):\n" + + " self.hours = hours\n" + + " self.minutes = minutes\n" + + " def utcoffset(self, dt):\n" + + " return timedelta(hours=self.hours, minutes=self.minutes)\n" + + " def tzname(self, dt):\n" + + " return \"GMT {0:00}:{1:00}\".format(self.hours, self.minutes)\n" + + " def dst (self, dt):\n" + + " return timedelta(0)\n").Handle; + + tzInfoCtor = Runtime.PyObject_GetAttrString(tzInfoMod, "GMT"); + if (tzInfoCtor == null) throw new PythonException(); } @@ -187,6 +218,14 @@ internal static IntPtr ToPython(object value, Type type) switch (tc) { case TypeCode.Object: + if (value is TimeSpan) + { + var timespan = (TimeSpan)value; + + IntPtr timeSpanArgs = Runtime.PyTuple_New(1); + Runtime.PyTuple_SetItem(timeSpanArgs, 0, Runtime.PyFloat_FromDouble(timespan.TotalDays)); + return Runtime.PyObject_CallObject(timeSpanCtor, timeSpanArgs); + } return CLRObject.GetInstHandle(value, type); case TypeCode.String: @@ -247,6 +286,21 @@ internal static IntPtr ToPython(object value, Type type) return Runtime.PyObject_CallObject(decimalCtor, decimalArgs); + case TypeCode.DateTime: + var datetime = (DateTime)value; + + IntPtr dateTimeArgs = Runtime.PyTuple_New(8); + Runtime.PyTuple_SetItem(dateTimeArgs, 0, Runtime.PyInt_FromInt32(datetime.Year)); + Runtime.PyTuple_SetItem(dateTimeArgs, 1, Runtime.PyInt_FromInt32(datetime.Month)); + Runtime.PyTuple_SetItem(dateTimeArgs, 2, Runtime.PyInt_FromInt32(datetime.Day)); + Runtime.PyTuple_SetItem(dateTimeArgs, 3, Runtime.PyInt_FromInt32(datetime.Hour)); + Runtime.PyTuple_SetItem(dateTimeArgs, 4, Runtime.PyInt_FromInt32(datetime.Minute)); + Runtime.PyTuple_SetItem(dateTimeArgs, 5, Runtime.PyInt_FromInt32(datetime.Second)); + Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(datetime.Millisecond)); + Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); + + return Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); + default: if (value is IEnumerable) { @@ -268,6 +322,16 @@ internal static IntPtr ToPython(object value, Type type) } } + private static IntPtr TzInfo(DateTimeKind kind) + { + if (kind == DateTimeKind.Unspecified) return Runtime.PyNone; + var offset = kind == DateTimeKind.Local ? DateTimeOffset.Now.Offset : TimeSpan.Zero; + IntPtr tzInfoArgs = Runtime.PyTuple_New(2); + Runtime.PyTuple_SetItem(tzInfoArgs, 0, Runtime.PyFloat_FromDouble(offset.Hours)); + Runtime.PyTuple_SetItem(tzInfoArgs, 1, Runtime.PyFloat_FromDouble(offset.Minutes)); + return Runtime.PyObject_CallObject(tzInfoCtor, tzInfoArgs); + } + /// /// In a few situations, we don't have any advisory type information @@ -490,6 +554,32 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo switch (tc) { + case TypeCode.Object: + if (obType == typeof(TimeSpan)) + { + op = Runtime.PyObject_Str(value); + TimeSpan ts; + var arr = Runtime.GetManagedString(op).Split(','); + string sts = arr.Length == 1 ? arr[0] : arr[1]; + if (!TimeSpan.TryParse(sts, out ts)) + { + goto type_error; + } + Runtime.XDecref(op); + + int days = 0; + if (arr.Length > 1) + { + if (!int.TryParse(arr[0].Split(' ')[0].Trim(), out days)) + { + goto type_error; + } + } + result = ts.Add(TimeSpan.FromDays(days)); + return true; + } + break; + case TypeCode.String: string st = Runtime.GetManagedString(value); if (st == null) @@ -851,6 +941,18 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo Runtime.XDecref(op); result = m; return true; + + case TypeCode.DateTime: + op = Runtime.PyObject_Str(value); + DateTime dt; + string sdt = Runtime.GetManagedString(op); + if (!DateTime.TryParse(sdt, out dt)) + { + goto type_error; + } + Runtime.XDecref(op); + result = dt; + return true; } From d2d12b3f5efb49625821c7a5922ef20ab6a277b3 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Fri, 13 Jul 2018 13:57:20 +0100 Subject: [PATCH 07/76] Fixes UTC conversion from python datetime to managed DateTime --- src/runtime/converter.cs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index c955a880f..481ee73c2 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -65,18 +65,18 @@ static Converter() timeSpanCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "timedelta"); if (timeSpanCtor == null) throw new PythonException(); - IntPtr tzInfoMod = PythonEngine.ModuleFromString("custom_tzinfo", - "from datetime import timedelta, tzinfo\n" + - "class GMT(tzinfo):\n" + - " def __init__(self, hours, minutes):\n" + - " self.hours = hours\n" + - " self.minutes = minutes\n" + - " def utcoffset(self, dt):\n" + - " return timedelta(hours=self.hours, minutes=self.minutes)\n" + - " def tzname(self, dt):\n" + - " return \"GMT {0:00}:{1:00}\".format(self.hours, self.minutes)\n" + - " def dst (self, dt):\n" + - " return timedelta(0)\n").Handle; + IntPtr tzInfoMod = PythonEngine.ModuleFromString("custom_tzinfo", @" +from datetime import timedelta, tzinfo +class GMT(tzinfo): + def __init__(self, hours, minutes): + self.hours = hours + self.minutes = minutes + def utcoffset(self, dt): + return timedelta(hours=self.hours, minutes=self.minutes) + def tzname(self, dt): + return f'GMT {self.hours:00}:{self.minutes:00}' + def dst (self, dt): + return timedelta(0)").Handle; tzInfoCtor = Runtime.PyObject_GetAttrString(tzInfoMod, "GMT"); if (tzInfoCtor == null) throw new PythonException(); @@ -951,7 +951,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo goto type_error; } Runtime.XDecref(op); - result = dt; + result = sdt.EndsWith("+00:00") ? dt.ToUniversalTime() : dt; return true; } From c6db86653e7fa4fa89e1f8404d72e346b67ed857 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Fri, 20 Jul 2018 15:58:10 -0300 Subject: [PATCH 08/76] Fixing memory leaks in the conversion of some types --- src/runtime/converter.cs | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 481ee73c2..ad8a8af02 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -1,8 +1,6 @@ using System; using System.Collections; -using System.Collections.Generic; using System.Globalization; -using System.Reflection; using System.Runtime.InteropServices; using System.Security; @@ -55,13 +53,13 @@ static Converter() IntPtr dateTimeMod = Runtime.PyImport_ImportModule("datetime"); if (dateTimeMod == null) throw new PythonException(); - + decimalCtor = Runtime.PyObject_GetAttrString(decimalMod, "Decimal"); if (decimalCtor == null) throw new PythonException(); - + dateTimeCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "datetime"); if (dateTimeCtor == null) throw new PythonException(); - + timeSpanCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "timedelta"); if (timeSpanCtor == null) throw new PythonException(); @@ -224,7 +222,10 @@ internal static IntPtr ToPython(object value, Type type) IntPtr timeSpanArgs = Runtime.PyTuple_New(1); Runtime.PyTuple_SetItem(timeSpanArgs, 0, Runtime.PyFloat_FromDouble(timespan.TotalDays)); - return Runtime.PyObject_CallObject(timeSpanCtor, timeSpanArgs); + var returnTimeSpan = Runtime.PyObject_CallObject(timeSpanCtor, timeSpanArgs); + // clean up + Runtime.XDecref(timeSpanArgs); + return returnTimeSpan; } return CLRObject.GetInstHandle(value, type); @@ -283,12 +284,14 @@ internal static IntPtr ToPython(object value, Type type) IntPtr d2p = Runtime.PyString_FromString(d2s); IntPtr decimalArgs = Runtime.PyTuple_New(1); Runtime.PyTuple_SetItem(decimalArgs, 0, d2p); - - return Runtime.PyObject_CallObject(decimalCtor, decimalArgs); + var returnDecimal = Runtime.PyObject_CallObject(decimalCtor, decimalArgs); + // clean up + Runtime.XDecref(decimalArgs); + return returnDecimal; case TypeCode.DateTime: var datetime = (DateTime)value; - + IntPtr dateTimeArgs = Runtime.PyTuple_New(8); Runtime.PyTuple_SetItem(dateTimeArgs, 0, Runtime.PyInt_FromInt32(datetime.Year)); Runtime.PyTuple_SetItem(dateTimeArgs, 1, Runtime.PyInt_FromInt32(datetime.Month)); @@ -298,8 +301,10 @@ internal static IntPtr ToPython(object value, Type type) Runtime.PyTuple_SetItem(dateTimeArgs, 5, Runtime.PyInt_FromInt32(datetime.Second)); Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(datetime.Millisecond)); Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); - - return Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); + var returnDateTime = Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); + // clean up + Runtime.XDecref(dateTimeArgs); + return returnDateTime; default: if (value is IEnumerable) @@ -329,7 +334,9 @@ private static IntPtr TzInfo(DateTimeKind kind) IntPtr tzInfoArgs = Runtime.PyTuple_New(2); Runtime.PyTuple_SetItem(tzInfoArgs, 0, Runtime.PyFloat_FromDouble(offset.Hours)); Runtime.PyTuple_SetItem(tzInfoArgs, 1, Runtime.PyFloat_FromDouble(offset.Minutes)); - return Runtime.PyObject_CallObject(tzInfoCtor, tzInfoArgs); + var returnValue = Runtime.PyObject_CallObject(tzInfoCtor, tzInfoArgs); + Runtime.XDecref(tzInfoArgs); + return returnValue; } From bec9563d2958acc5adf3e8972b23609fe3914402 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 23 Aug 2018 23:27:02 +0100 Subject: [PATCH 09/76] Do not include timezone if DateTimeKind is Unspecified DECREF'ing datetime timezone argument when DateTimeKind is Unspecified was causing `Fatal Python error: deallocating None` because the object was set to `Runtime.PyNone`. Fixed the input to datetime constructor as we were passing milliseconds, where it should be microseconds. --- src/runtime/converter.cs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index ad8a8af02..a33ade002 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -292,15 +292,27 @@ internal static IntPtr ToPython(object value, Type type) case TypeCode.DateTime: var datetime = (DateTime)value; - IntPtr dateTimeArgs = Runtime.PyTuple_New(8); + var size = datetime.Kind == DateTimeKind.Unspecified ? 7 : 8; + + IntPtr dateTimeArgs = Runtime.PyTuple_New(size); Runtime.PyTuple_SetItem(dateTimeArgs, 0, Runtime.PyInt_FromInt32(datetime.Year)); Runtime.PyTuple_SetItem(dateTimeArgs, 1, Runtime.PyInt_FromInt32(datetime.Month)); Runtime.PyTuple_SetItem(dateTimeArgs, 2, Runtime.PyInt_FromInt32(datetime.Day)); Runtime.PyTuple_SetItem(dateTimeArgs, 3, Runtime.PyInt_FromInt32(datetime.Hour)); Runtime.PyTuple_SetItem(dateTimeArgs, 4, Runtime.PyInt_FromInt32(datetime.Minute)); Runtime.PyTuple_SetItem(dateTimeArgs, 5, Runtime.PyInt_FromInt32(datetime.Second)); - Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(datetime.Millisecond)); - Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); + + // datetime.datetime 6th argument represents micro seconds + var totalSeconds = datetime.TimeOfDay.TotalSeconds; + var microSeconds = Convert.ToInt32((totalSeconds - Math.Truncate(totalSeconds)) * 1000000); + if (microSeconds == 1000000) microSeconds = 999999; + Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(microSeconds)); + + if (size == 8) + { + Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); + } + var returnDateTime = Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); // clean up Runtime.XDecref(dateTimeArgs); From 76695b4b4ca474214fb6a4ab8a0237c7607997d4 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Mon, 27 Aug 2018 18:05:38 +0100 Subject: [PATCH 10/76] Sets the version to 1.0.5.12 to match QuantConnect's nuget one. --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ac1d31324..8a1c05306 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.4.0.dev0 +current_version = 1.0.5.12 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 3641185bb..cf9bb4749 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "2.4.0.dev0" + version: "1.0.5.12" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 4ec2a2113..d35041876 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="2.4.0.dev0", + version="1.0.5.12", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index c164e75d6..4a3b83782 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("2.4.0")] +[assembly: AssemblyVersion("1.0.5.12")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 3bb3a533b..c5b789a18 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("2.4.0"), + Version = new Version("1.0.5.12"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index e708a54ac..7217f98d1 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "2.4.0.dev0" +__version__ = "1.0.5.12" class clrproperty(object): From 7e655cebcc493813ab69940ec21bab73327845db Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Mon, 14 Jan 2019 12:18:30 -0300 Subject: [PATCH 11/76] C# decimal conversion - C# decimal conversion will use C# double and python float due to the big performance impact of converting C# decimal to python decimal. --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/runtime/converter.cs | 18 +++--------------- src/runtime/resources/clr.py | 2 +- 6 files changed, 8 insertions(+), 20 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8a1c05306..2518f77fb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.12 +current_version = 1.0.5.13 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index cf9bb4749..3aa2d1a62 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.12" + version: "1.0.5.13" build: skip: True # [not win] diff --git a/setup.py b/setup.py index d35041876..3695e33fa 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.12", + version="1.0.5.13", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index 4a3b83782..e9d39ba33 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.12")] +[assembly: AssemblyVersion("1.0.5.13")] diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index a33ade002..d8ec3fcd2 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -28,7 +28,6 @@ private Converter() private static Type flagsType; private static Type boolType; private static Type typeType; - private static IntPtr decimalCtor; private static IntPtr dateTimeCtor; private static IntPtr timeSpanCtor; private static IntPtr tzInfoCtor; @@ -48,15 +47,9 @@ static Converter() boolType = typeof(Boolean); typeType = typeof(Type); - IntPtr decimalMod = Runtime.PyImport_ImportModule("decimal"); - if (decimalMod == null) throw new PythonException(); - IntPtr dateTimeMod = Runtime.PyImport_ImportModule("datetime"); if (dateTimeMod == null) throw new PythonException(); - decimalCtor = Runtime.PyObject_GetAttrString(decimalMod, "Decimal"); - if (decimalCtor == null) throw new PythonException(); - dateTimeCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "datetime"); if (dateTimeCtor == null) throw new PythonException(); @@ -280,14 +273,9 @@ internal static IntPtr ToPython(object value, Type type) return Runtime.PyLong_FromUnsignedLongLong((ulong)value); case TypeCode.Decimal: - string d2s = ((decimal)value).ToString(nfi); - IntPtr d2p = Runtime.PyString_FromString(d2s); - IntPtr decimalArgs = Runtime.PyTuple_New(1); - Runtime.PyTuple_SetItem(decimalArgs, 0, d2p); - var returnDecimal = Runtime.PyObject_CallObject(decimalCtor, decimalArgs); - // clean up - Runtime.XDecref(decimalArgs); - return returnDecimal; + // C# decimal to python decimal has a big impact on performance + // so we will use C# double and python float + return Runtime.PyFloat_FromDouble(decimal.ToDouble((decimal) value)); case TypeCode.DateTime: var datetime = (DateTime)value; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 7217f98d1..7caff08f4 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.12" +__version__ = "1.0.5.13" class clrproperty(object): From 3511f63cc69134181808947f10fb68326aecbf36 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 28 Sep 2017 23:24:38 +0100 Subject: [PATCH 12/76] Implements Nullable support --- src/runtime/converter.cs | 8 +++++++- src/runtime/methodbinder.cs | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 11c67bf82..ec367015b 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -437,6 +437,12 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return false; } + var underlyingType = Nullable.GetUnderlyingType(obType); + if (underlyingType != null) + { + return ToManagedValue(value, underlyingType, out result, setError); + } + return ToPrimitive(value, obType, out result, setError); } @@ -846,7 +852,7 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s var size = Runtime.PySequence_Size(value); result = null; - if (size < 0) + if (size < 0 || elementType.IsGenericType) { if (setError) { diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 5e800c36f..7d24cde9a 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -394,8 +394,14 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth } if (!typematch) { + // this takes care of nullables + var underlyingType = Nullable.GetUnderlyingType(pi[n].ParameterType); + if (underlyingType == null) + { + underlyingType = pi[n].ParameterType; + } // this takes care of enum values - TypeCode argtypecode = Type.GetTypeCode(pi[n].ParameterType); + TypeCode argtypecode = Type.GetTypeCode(underlyingType); TypeCode paramtypecode = Type.GetTypeCode(clrtype); if (argtypecode == paramtypecode) { From d24b7eec40b1c136a5dc37561ede1e7e7c8836ef Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 28 Sep 2017 23:28:39 +0100 Subject: [PATCH 13/76] Implements Implicit Conversion --- src/runtime/converter.cs | 14 ++++++++++++++ src/runtime/methodbinder.cs | 7 +++++++ 2 files changed, 21 insertions(+) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index ec367015b..35a6dc1b8 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -443,6 +443,20 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return ToManagedValue(value, underlyingType, out result, setError); } + var opImplicit = obType.GetMethod("op_Implicit", new[] { obType }); + if (opImplicit != null) + { + if (ToManagedValue(value, opImplicit.ReturnType, out result, setError)) + { + opImplicit = obType.GetMethod("op_Implicit", new[] { result.GetType() }); + if (opImplicit != null) + { + result = opImplicit.Invoke(null, new[] { result }); + } + return opImplicit != null; + } + } + return ToPrimitive(value, obType, out result, setError); } diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 7d24cde9a..0c5433076 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -408,6 +408,13 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth typematch = true; clrtype = pi[n].ParameterType; } + // this takes care of implicit conversions + var opImplicit = pi[n].ParameterType.GetMethod("op_Implicit", new[] { clrtype }); + if (opImplicit != null) + { + typematch = opImplicit.ReturnType == pi[n].ParameterType; + clrtype = pi[n].ParameterType; + } } Runtime.XDecref(pyoptype); if (!typematch) From 2a6c1aa06fcf3519bfb0b0ebcc0f077802386d3d Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Mon, 2 Apr 2018 11:53:48 +0100 Subject: [PATCH 14/76] Adds method name to "no method matches" error --- src/runtime/methodbinder.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 0c5433076..c33015725 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -186,6 +186,13 @@ internal static int GetPrecedence(MethodBase mi) val += ArgPrecedence(pi[i].ParameterType); } + var info = mi as MethodInfo; + if (info != null) + { + val += ArgPrecedence(info.ReturnType); + val += mi.DeclaringType == mi.ReflectedType ? 0 : 3000; + } + return val; } @@ -200,6 +207,11 @@ internal static int ArgPrecedence(Type t) return 3000; } + if (t.IsAssignableFrom(typeof(PyObject))) + { + return -1; + } + TypeCode tc = Type.GetTypeCode(t); // TODO: Clean up switch (tc) From a6f2d561db67b0c0291a61767788a228fb148090 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 28 Sep 2017 23:01:37 +0100 Subject: [PATCH 15/76] Implements System.Decimal support Convertes System.Decimal to decimal.decimal and vice-versa --- src/runtime/converter.cs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 35a6dc1b8..77657e00a 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -31,6 +31,7 @@ private Converter() private static Type flagsType; private static Type boolType; private static Type typeType; + private static IntPtr decimalCtor; static Converter() { @@ -46,6 +47,9 @@ static Converter() flagsType = typeof(FlagsAttribute); boolType = typeof(Boolean); typeType = typeof(Type); + + IntPtr decimalMod = Runtime.PyImport_ImportModule("decimal"); + if (decimalMod == null) throw new PythonException(); } @@ -101,6 +105,9 @@ internal static IntPtr GetPythonTypeByAlias(Type op) if (op == boolType) return Runtime.PyBoolType; + if (op == decimalType) + return Runtime.PyFloatType; + return IntPtr.Zero; } @@ -230,6 +237,14 @@ internal static IntPtr ToPython(object value, Type type) case TypeCode.UInt64: return Runtime.PyLong_FromUnsignedLongLong((ulong)value); + case TypeCode.Decimal: + string d2s = ((decimal)value).ToString(nfi); + IntPtr d2p = Runtime.PyString_FromString(d2s); + IntPtr decimalArgs = Runtime.PyTuple_New(1); + Runtime.PyTuple_SetItem(decimalArgs, 0, d2p); + + return Runtime.PyObject_CallObject(decimalCtor, decimalArgs); + default: if (value is IEnumerable) { @@ -822,6 +837,18 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo Runtime.XDecref(op); result = d; return true; + + case TypeCode.Decimal: + op = Runtime.PyObject_Str(value); + decimal m; + string sm = Runtime.GetManagedString(op); + if (!Decimal.TryParse(sm, NumberStyles.Number, nfi, out m)) + { + goto type_error; + } + Runtime.XDecref(op); + result = m; + return true; } From a7b169dd76ec61a86ef36db4864db47773325e74 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Mon, 2 Apr 2018 11:51:09 +0100 Subject: [PATCH 16/76] Fixes decimal support --- src/runtime/converter.cs | 5 ++++- src/runtime/methodbinder.cs | 6 ++++++ src/runtime/runtime.cs | 9 +++++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 77657e00a..6f16ee4b7 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -76,6 +76,9 @@ internal static Type GetTypeByAlias(IntPtr op) if (op == Runtime.PyBoolType) return boolType; + if (op == Runtime.PyDecimalType) + return decimalType; + return null; } @@ -106,7 +109,7 @@ internal static IntPtr GetPythonTypeByAlias(Type op) return Runtime.PyBoolType; if (op == decimalType) - return Runtime.PyFloatType; + return Runtime.PyDecimalType; return IntPtr.Zero; } diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index c33015725..b608b6c55 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -420,6 +420,12 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth typematch = true; clrtype = pi[n].ParameterType; } + // accepts non-decimal numbers in decimal parameters + if (underlyingType == typeof(decimal)) + { + clrtype = pi[n].ParameterType; + typematch = Converter.ToManaged(op, clrtype, out arg, false); + } // this takes care of implicit conversions var opImplicit = pi[n].ParameterType.GetMethod("op_Implicit", new[] { clrtype }); if (opImplicit != null) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 863eb4034..21a1d20fd 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -368,6 +368,14 @@ internal static void Initialize(bool initSigs = false) PyFloatType = PyObject_Type(op); XDecref(op); + IntPtr decimalMod = PyImport_ImportModule("decimal"); + IntPtr decimalCtor = PyObject_GetAttrString(decimalMod, "Decimal"); + op = PyObject_CallObject(decimalCtor, IntPtr.Zero); + PyDecimalType = PyObject_Type(op); + XDecref(op); + XDecref(decimalMod); + XDecref(decimalCtor); + #if PYTHON3 PyClassType = IntPtr.Zero; PyInstanceType = IntPtr.Zero; @@ -512,6 +520,7 @@ internal static int AtExit() internal static IntPtr PyBoolType; internal static IntPtr PyNoneType; internal static IntPtr PyTypeType; + internal static IntPtr PyDecimalType; #if PYTHON3 internal static IntPtr PyBytesType; From ba5c2f660e90ee256acaa72664755e060448884e Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 28 Sep 2017 23:17:41 +0100 Subject: [PATCH 17/76] Implements System.DateTime support Converts System.DateTime and System.TimeSpan to datetime.datetime and datetime.timedelta and vice-versa --- src/runtime/converter.cs | 102 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 6f16ee4b7..52e06498e 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -32,6 +32,9 @@ private Converter() private static Type boolType; private static Type typeType; private static IntPtr decimalCtor; + private static IntPtr dateTimeCtor; + private static IntPtr timeSpanCtor; + private static IntPtr tzInfoCtor; static Converter() { @@ -50,6 +53,34 @@ static Converter() IntPtr decimalMod = Runtime.PyImport_ImportModule("decimal"); if (decimalMod == null) throw new PythonException(); + + IntPtr dateTimeMod = Runtime.PyImport_ImportModule("datetime"); + if (dateTimeMod == null) throw new PythonException(); + + decimalCtor = Runtime.PyObject_GetAttrString(decimalMod, "Decimal"); + if (decimalCtor == null) throw new PythonException(); + + dateTimeCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "datetime"); + if (dateTimeCtor == null) throw new PythonException(); + + timeSpanCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "timedelta"); + if (timeSpanCtor == null) throw new PythonException(); + + IntPtr tzInfoMod = PythonEngine.ModuleFromString("custom_tzinfo", + "from datetime import timedelta, tzinfo\n" + + "class GMT(tzinfo):\n" + + " def __init__(self, hours, minutes):\n" + + " self.hours = hours\n" + + " self.minutes = minutes\n" + + " def utcoffset(self, dt):\n" + + " return timedelta(hours=self.hours, minutes=self.minutes)\n" + + " def tzname(self, dt):\n" + + " return \"GMT {0:00}:{1:00}\".format(self.hours, self.minutes)\n" + + " def dst (self, dt):\n" + + " return timedelta(0)\n").Handle; + + tzInfoCtor = Runtime.PyObject_GetAttrString(tzInfoMod, "GMT"); + if (tzInfoCtor == null) throw new PythonException(); } @@ -188,6 +219,14 @@ internal static IntPtr ToPython(object value, Type type) switch (tc) { case TypeCode.Object: + if (value is TimeSpan) + { + var timespan = (TimeSpan)value; + + IntPtr timeSpanArgs = Runtime.PyTuple_New(1); + Runtime.PyTuple_SetItem(timeSpanArgs, 0, Runtime.PyFloat_FromDouble(timespan.TotalDays)); + return Runtime.PyObject_CallObject(timeSpanCtor, timeSpanArgs); + } return CLRObject.GetInstHandle(value, type); case TypeCode.String: @@ -248,6 +287,21 @@ internal static IntPtr ToPython(object value, Type type) return Runtime.PyObject_CallObject(decimalCtor, decimalArgs); + case TypeCode.DateTime: + var datetime = (DateTime)value; + + IntPtr dateTimeArgs = Runtime.PyTuple_New(8); + Runtime.PyTuple_SetItem(dateTimeArgs, 0, Runtime.PyInt_FromInt32(datetime.Year)); + Runtime.PyTuple_SetItem(dateTimeArgs, 1, Runtime.PyInt_FromInt32(datetime.Month)); + Runtime.PyTuple_SetItem(dateTimeArgs, 2, Runtime.PyInt_FromInt32(datetime.Day)); + Runtime.PyTuple_SetItem(dateTimeArgs, 3, Runtime.PyInt_FromInt32(datetime.Hour)); + Runtime.PyTuple_SetItem(dateTimeArgs, 4, Runtime.PyInt_FromInt32(datetime.Minute)); + Runtime.PyTuple_SetItem(dateTimeArgs, 5, Runtime.PyInt_FromInt32(datetime.Second)); + Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(datetime.Millisecond)); + Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); + + return Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); + default: if (value is IEnumerable) { @@ -269,6 +323,16 @@ internal static IntPtr ToPython(object value, Type type) } } + private static IntPtr TzInfo(DateTimeKind kind) + { + if (kind == DateTimeKind.Unspecified) return Runtime.PyNone; + var offset = kind == DateTimeKind.Local ? DateTimeOffset.Now.Offset : TimeSpan.Zero; + IntPtr tzInfoArgs = Runtime.PyTuple_New(2); + Runtime.PyTuple_SetItem(tzInfoArgs, 0, Runtime.PyFloat_FromDouble(offset.Hours)); + Runtime.PyTuple_SetItem(tzInfoArgs, 1, Runtime.PyFloat_FromDouble(offset.Minutes)); + return Runtime.PyObject_CallObject(tzInfoCtor, tzInfoArgs); + } + /// /// In a few situations, we don't have any advisory type information @@ -491,6 +555,32 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo switch (tc) { + case TypeCode.Object: + if (obType == typeof(TimeSpan)) + { + op = Runtime.PyObject_Str(value); + TimeSpan ts; + var arr = Runtime.GetManagedString(op).Split(','); + string sts = arr.Length == 1 ? arr[0] : arr[1]; + if (!TimeSpan.TryParse(sts, out ts)) + { + goto type_error; + } + Runtime.XDecref(op); + + int days = 0; + if (arr.Length > 1) + { + if (!int.TryParse(arr[0].Split(' ')[0].Trim(), out days)) + { + goto type_error; + } + } + result = ts.Add(TimeSpan.FromDays(days)); + return true; + } + break; + case TypeCode.String: string st = Runtime.GetManagedString(value); if (st == null) @@ -852,6 +942,18 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo Runtime.XDecref(op); result = m; return true; + + case TypeCode.DateTime: + op = Runtime.PyObject_Str(value); + DateTime dt; + string sdt = Runtime.GetManagedString(op); + if (!DateTime.TryParse(sdt, out dt)) + { + goto type_error; + } + Runtime.XDecref(op); + result = dt; + return true; } From 15e4f9f1144513e4f176d65dc18dea4020d49db5 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Fri, 13 Jul 2018 13:57:20 +0100 Subject: [PATCH 18/76] Fixes UTC conversion from python datetime to managed DateTime --- src/runtime/converter.cs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 52e06498e..4063572e5 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -66,18 +66,18 @@ static Converter() timeSpanCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "timedelta"); if (timeSpanCtor == null) throw new PythonException(); - IntPtr tzInfoMod = PythonEngine.ModuleFromString("custom_tzinfo", - "from datetime import timedelta, tzinfo\n" + - "class GMT(tzinfo):\n" + - " def __init__(self, hours, minutes):\n" + - " self.hours = hours\n" + - " self.minutes = minutes\n" + - " def utcoffset(self, dt):\n" + - " return timedelta(hours=self.hours, minutes=self.minutes)\n" + - " def tzname(self, dt):\n" + - " return \"GMT {0:00}:{1:00}\".format(self.hours, self.minutes)\n" + - " def dst (self, dt):\n" + - " return timedelta(0)\n").Handle; + IntPtr tzInfoMod = PythonEngine.ModuleFromString("custom_tzinfo", @" +from datetime import timedelta, tzinfo +class GMT(tzinfo): + def __init__(self, hours, minutes): + self.hours = hours + self.minutes = minutes + def utcoffset(self, dt): + return timedelta(hours=self.hours, minutes=self.minutes) + def tzname(self, dt): + return f'GMT {self.hours:00}:{self.minutes:00}' + def dst (self, dt): + return timedelta(0)").Handle; tzInfoCtor = Runtime.PyObject_GetAttrString(tzInfoMod, "GMT"); if (tzInfoCtor == null) throw new PythonException(); @@ -952,7 +952,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo goto type_error; } Runtime.XDecref(op); - result = dt; + result = sdt.EndsWith("+00:00") ? dt.ToUniversalTime() : dt; return true; } From 349bcd710c07a932296b5115b147152e9279e2c7 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Fri, 20 Jul 2018 15:58:10 -0300 Subject: [PATCH 19/76] Fixing memory leaks in the conversion of some types --- src/runtime/converter.cs | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 4063572e5..fa5bf729b 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -1,8 +1,6 @@ using System; using System.Collections; -using System.Collections.Generic; using System.Globalization; -using System.Reflection; using System.Runtime.InteropServices; using System.Security; using System.ComponentModel; @@ -56,13 +54,13 @@ static Converter() IntPtr dateTimeMod = Runtime.PyImport_ImportModule("datetime"); if (dateTimeMod == null) throw new PythonException(); - + decimalCtor = Runtime.PyObject_GetAttrString(decimalMod, "Decimal"); if (decimalCtor == null) throw new PythonException(); - + dateTimeCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "datetime"); if (dateTimeCtor == null) throw new PythonException(); - + timeSpanCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "timedelta"); if (timeSpanCtor == null) throw new PythonException(); @@ -225,7 +223,10 @@ internal static IntPtr ToPython(object value, Type type) IntPtr timeSpanArgs = Runtime.PyTuple_New(1); Runtime.PyTuple_SetItem(timeSpanArgs, 0, Runtime.PyFloat_FromDouble(timespan.TotalDays)); - return Runtime.PyObject_CallObject(timeSpanCtor, timeSpanArgs); + var returnTimeSpan = Runtime.PyObject_CallObject(timeSpanCtor, timeSpanArgs); + // clean up + Runtime.XDecref(timeSpanArgs); + return returnTimeSpan; } return CLRObject.GetInstHandle(value, type); @@ -284,12 +285,14 @@ internal static IntPtr ToPython(object value, Type type) IntPtr d2p = Runtime.PyString_FromString(d2s); IntPtr decimalArgs = Runtime.PyTuple_New(1); Runtime.PyTuple_SetItem(decimalArgs, 0, d2p); - - return Runtime.PyObject_CallObject(decimalCtor, decimalArgs); + var returnDecimal = Runtime.PyObject_CallObject(decimalCtor, decimalArgs); + // clean up + Runtime.XDecref(decimalArgs); + return returnDecimal; case TypeCode.DateTime: var datetime = (DateTime)value; - + IntPtr dateTimeArgs = Runtime.PyTuple_New(8); Runtime.PyTuple_SetItem(dateTimeArgs, 0, Runtime.PyInt_FromInt32(datetime.Year)); Runtime.PyTuple_SetItem(dateTimeArgs, 1, Runtime.PyInt_FromInt32(datetime.Month)); @@ -299,8 +302,10 @@ internal static IntPtr ToPython(object value, Type type) Runtime.PyTuple_SetItem(dateTimeArgs, 5, Runtime.PyInt_FromInt32(datetime.Second)); Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(datetime.Millisecond)); Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); - - return Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); + var returnDateTime = Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); + // clean up + Runtime.XDecref(dateTimeArgs); + return returnDateTime; default: if (value is IEnumerable) @@ -330,7 +335,9 @@ private static IntPtr TzInfo(DateTimeKind kind) IntPtr tzInfoArgs = Runtime.PyTuple_New(2); Runtime.PyTuple_SetItem(tzInfoArgs, 0, Runtime.PyFloat_FromDouble(offset.Hours)); Runtime.PyTuple_SetItem(tzInfoArgs, 1, Runtime.PyFloat_FromDouble(offset.Minutes)); - return Runtime.PyObject_CallObject(tzInfoCtor, tzInfoArgs); + var returnValue = Runtime.PyObject_CallObject(tzInfoCtor, tzInfoArgs); + Runtime.XDecref(tzInfoArgs); + return returnValue; } From ad8a585387c95248fb3098aa611042adbb797143 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Thu, 23 Aug 2018 23:27:02 +0100 Subject: [PATCH 20/76] Do not include timezone if DateTimeKind is Unspecified DECREF'ing datetime timezone argument when DateTimeKind is Unspecified was causing `Fatal Python error: deallocating None` because the object was set to `Runtime.PyNone`. Fixed the input to datetime constructor as we were passing milliseconds, where it should be microseconds. --- src/runtime/converter.cs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index fa5bf729b..97450be77 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -293,15 +293,27 @@ internal static IntPtr ToPython(object value, Type type) case TypeCode.DateTime: var datetime = (DateTime)value; - IntPtr dateTimeArgs = Runtime.PyTuple_New(8); + var size = datetime.Kind == DateTimeKind.Unspecified ? 7 : 8; + + IntPtr dateTimeArgs = Runtime.PyTuple_New(size); Runtime.PyTuple_SetItem(dateTimeArgs, 0, Runtime.PyInt_FromInt32(datetime.Year)); Runtime.PyTuple_SetItem(dateTimeArgs, 1, Runtime.PyInt_FromInt32(datetime.Month)); Runtime.PyTuple_SetItem(dateTimeArgs, 2, Runtime.PyInt_FromInt32(datetime.Day)); Runtime.PyTuple_SetItem(dateTimeArgs, 3, Runtime.PyInt_FromInt32(datetime.Hour)); Runtime.PyTuple_SetItem(dateTimeArgs, 4, Runtime.PyInt_FromInt32(datetime.Minute)); Runtime.PyTuple_SetItem(dateTimeArgs, 5, Runtime.PyInt_FromInt32(datetime.Second)); - Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(datetime.Millisecond)); - Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); + + // datetime.datetime 6th argument represents micro seconds + var totalSeconds = datetime.TimeOfDay.TotalSeconds; + var microSeconds = Convert.ToInt32((totalSeconds - Math.Truncate(totalSeconds)) * 1000000); + if (microSeconds == 1000000) microSeconds = 999999; + Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(microSeconds)); + + if (size == 8) + { + Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); + } + var returnDateTime = Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); // clean up Runtime.XDecref(dateTimeArgs); From c5019a3d093c3aef1d85a2fc04ee0aac7b34b101 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Mon, 27 Aug 2018 18:05:38 +0100 Subject: [PATCH 21/76] Sets the version to 1.0.5.12 to match QuantConnect's nuget one. --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ac1d31324..8a1c05306 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.4.0.dev0 +current_version = 1.0.5.12 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 3641185bb..cf9bb4749 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "2.4.0.dev0" + version: "1.0.5.12" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 1b6f07ea6..309ece49a 100644 --- a/setup.py +++ b/setup.py @@ -492,7 +492,7 @@ def run(self): setup( name="pythonnet", - version="2.4.0.dev0", + version="1.0.5.12", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index c164e75d6..4a3b83782 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("2.4.0")] +[assembly: AssemblyVersion("1.0.5.12")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 3bb3a533b..c5b789a18 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("2.4.0"), + Version = new Version("1.0.5.12"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index e708a54ac..7217f98d1 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "2.4.0.dev0" +__version__ = "1.0.5.12" class clrproperty(object): From 1f7d30854485dad3b5ae59fc7db17cb836e7923f Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Mon, 14 Jan 2019 12:18:30 -0300 Subject: [PATCH 22/76] C# decimal conversion - C# decimal conversion will use C# double and python float due to the big performance impact of converting C# decimal to python decimal. --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/runtime/converter.cs | 18 +++--------------- src/runtime/resources/clr.py | 2 +- 6 files changed, 8 insertions(+), 20 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8a1c05306..2518f77fb 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.12 +current_version = 1.0.5.13 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index cf9bb4749..3aa2d1a62 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.12" + version: "1.0.5.13" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 309ece49a..10f1b178b 100644 --- a/setup.py +++ b/setup.py @@ -492,7 +492,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.12", + version="1.0.5.13", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index 4a3b83782..e9d39ba33 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.12")] +[assembly: AssemblyVersion("1.0.5.13")] diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 97450be77..629eab4df 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -29,7 +29,6 @@ private Converter() private static Type flagsType; private static Type boolType; private static Type typeType; - private static IntPtr decimalCtor; private static IntPtr dateTimeCtor; private static IntPtr timeSpanCtor; private static IntPtr tzInfoCtor; @@ -49,15 +48,9 @@ static Converter() boolType = typeof(Boolean); typeType = typeof(Type); - IntPtr decimalMod = Runtime.PyImport_ImportModule("decimal"); - if (decimalMod == null) throw new PythonException(); - IntPtr dateTimeMod = Runtime.PyImport_ImportModule("datetime"); if (dateTimeMod == null) throw new PythonException(); - decimalCtor = Runtime.PyObject_GetAttrString(decimalMod, "Decimal"); - if (decimalCtor == null) throw new PythonException(); - dateTimeCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "datetime"); if (dateTimeCtor == null) throw new PythonException(); @@ -281,14 +274,9 @@ internal static IntPtr ToPython(object value, Type type) return Runtime.PyLong_FromUnsignedLongLong((ulong)value); case TypeCode.Decimal: - string d2s = ((decimal)value).ToString(nfi); - IntPtr d2p = Runtime.PyString_FromString(d2s); - IntPtr decimalArgs = Runtime.PyTuple_New(1); - Runtime.PyTuple_SetItem(decimalArgs, 0, d2p); - var returnDecimal = Runtime.PyObject_CallObject(decimalCtor, decimalArgs); - // clean up - Runtime.XDecref(decimalArgs); - return returnDecimal; + // C# decimal to python decimal has a big impact on performance + // so we will use C# double and python float + return Runtime.PyFloat_FromDouble(decimal.ToDouble((decimal) value)); case TypeCode.DateTime: var datetime = (DateTime)value; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 7217f98d1..7caff08f4 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.12" +__version__ = "1.0.5.13" class clrproperty(object): From 54f284bac406152290955f4e8af9cd8a3a5d3c11 Mon Sep 17 00:00:00 2001 From: Alexandre Catarino Date: Fri, 18 Jan 2019 21:17:27 +0000 Subject: [PATCH 23/76] Revert "Rebase to pythonnet master" --- .travis.yml | 39 +- AUTHORS.md | 5 - CHANGELOG.md | 20 - NuGet.config | 2 +- appveyor.yml | 9 - setup.py | 17 +- src/embed_tests/GlobalTestsSetup.cs | 21 - .../Python.EmbeddingTest.15.csproj | 1 - src/embed_tests/Python.EmbeddingTest.csproj | 7 +- src/embed_tests/TestConverter.cs | 2 +- src/embed_tests/TestDomainReload.cs | 239 ---------- src/embed_tests/TestNamedArguments.cs | 2 +- src/embed_tests/TestPyObject.cs | 2 +- src/embed_tests/TestPySequence.cs | 6 +- src/embed_tests/TestPyWith.cs | 2 +- src/embed_tests/TestPythonEngineProperties.cs | 39 +- src/embed_tests/TestRuntime.cs | 34 +- src/embed_tests/TestTypeManager.cs | 65 --- src/embed_tests/dynamic.cs | 17 +- src/embed_tests/pyinitialize.cs | 60 --- src/runtime/Python.Runtime.15.csproj | 4 +- src/runtime/Python.Runtime.csproj | 13 +- src/runtime/Util.cs | 2 +- src/runtime/arrayobject.cs | 4 +- src/runtime/assemblymanager.cs | 56 +-- src/runtime/classbase.cs | 18 + src/runtime/classderived.cs | 9 +- src/runtime/classmanager.cs | 5 - src/runtime/classobject.cs | 6 +- src/runtime/clrobject.cs | 2 +- src/runtime/converter.cs | 7 +- src/runtime/debughelper.cs | 6 +- src/runtime/exceptions.cs | 7 +- src/runtime/extensiontype.cs | 21 + src/runtime/genericutil.cs | 6 - src/runtime/importhook.cs | 41 +- src/runtime/indexer.cs | 6 +- src/runtime/interfaceobject.cs | 2 +- src/runtime/interop37.cs | 149 ------- src/runtime/metatype.cs | 2 +- src/runtime/methodbinder.cs | 23 +- src/runtime/methodbinding.cs | 4 +- src/runtime/moduleobject.cs | 22 +- src/runtime/pyobject.cs | 4 +- src/runtime/pyscope.cs | 7 +- src/runtime/pythonengine.cs | 102 +---- src/runtime/runtime.cs | 409 +++--------------- src/runtime/typemanager.cs | 273 +----------- src/testing/InheritanceTest.cs | 14 - src/testing/Python.Test.csproj | 3 +- src/testing/conversiontest.cs | 15 - src/tests/importtest.py | 13 - src/tests/test_class.py | 11 - src/tests/test_conversion.py | 13 +- src/tests/test_exceptions.py | 3 +- src/tests/test_import.py | 11 +- src/tests/test_subclass.py | 33 -- tools/geninterop/fake_libc_include/crypt.h | 1 - tools/geninterop/geninterop.py | 5 +- 59 files changed, 220 insertions(+), 1701 deletions(-) delete mode 100644 src/embed_tests/GlobalTestsSetup.cs delete mode 100644 src/embed_tests/TestDomainReload.cs delete mode 100644 src/embed_tests/TestTypeManager.cs delete mode 100644 src/runtime/interop37.cs delete mode 100644 src/testing/InheritanceTest.cs delete mode 100644 src/tests/importtest.py delete mode 100644 tools/geninterop/fake_libc_include/crypt.h diff --git a/.travis.yml b/.travis.yml index d059bdcde..900e207a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,6 @@ matrix: - dotnet-hostfxr-2.0.0 - dotnet-runtime-2.0.0 - dotnet-sdk-2.0.0 - - python: 3.4 env: *xplat-env addons: *xplat-addons @@ -34,24 +33,9 @@ matrix: - python: 3.6 env: *xplat-env addons: *xplat-addons - - - python: 3.7 + - python: "3.7-dev" env: *xplat-env - dist: xenial - sudo: true - addons: &xplat-addons-xenial - apt: - sources: - - sourceline: deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod xenial main - key_url: https://packages.microsoft.com/keys/microsoft.asc - - sourceline: deb https://download.mono-project.com/repo/ubuntu stable-xenial main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF - packages: - - mono-devel - - ca-certificates-mono - - dotnet-hostfxr-2.0.0 - - dotnet-runtime-2.0.0 - - dotnet-sdk-2.0.0 + addons: *xplat-addons # --------------------- Classic builds ------------------------ - python: 2.7 @@ -68,18 +52,15 @@ matrix: - python: 3.6 env: *classic-env - - python: 3.7 + - python: "3.7-dev" + env: *classic-env + + allow_failures: + - python: "3.7-dev" + env: *xplat-env + + - python: "3.7-dev" env: *classic-env - dist: xenial - sudo: true - addons: - apt: - sources: - - sourceline: deb http://download.mono-project.com/repo/ubuntu xenial main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF - packages: - - mono-devel - - ca-certificates-mono env: global: diff --git a/AUTHORS.md b/AUTHORS.md index fe2d2b172..3c39794e4 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -14,7 +14,6 @@ - Alexandre Catarino([@AlexCatarino](https://github.com/AlexCatarino)) - Arvid JB ([@ArvidJB](https://github.com/ArvidJB)) -- Benoît Hudson ([@benoithudson](https://github.com/benoithudson)) - Bradley Friedman ([@leith-bartrich](https://github.com/leith-bartrich)) - Callum Noble ([@callumnoble](https://github.com/callumnoble)) - Christian Heimes ([@tiran](https://github.com/tiran)) @@ -23,12 +22,10 @@ - Daniel Fernandez ([@fdanny](https://github.com/fdanny)) - Daniel Santana ([@dgsantana](https://github.com/dgsantana)) - Dave Hirschfeld ([@dhirschfeld](https://github.com/dhirschfeld)) -- David Lassonde ([@lassond](https://github.com/lassond)) - David Lechner ([@dlech](https://github.com/dlech)) - Dmitriy Se ([@dmitriyse](https://github.com/dmitriyse)) - He-chien Tsai ([@t3476](https://github.com/t3476)) -   Ivan Cronyn ([@cronan](https://github.com/cronan)) -- Jan Krivanek ([@jakrivan](https://github.com/jakrivan)) -   Jeff Reback ([@jreback](https://github.com/jreback)) - Joe Frayne ([@jfrayne](https://github.com/jfrayne)) - John Burnett ([@johnburnett](https://github.com/johnburnett)) @@ -42,8 +39,6 @@ - Sam Winstanley ([@swinstanley](https://github.com/swinstanley)) - Sean Freitag ([@cowboygneox](https://github.com/cowboygneox)) - Serge Weinstock ([@sweinst](https://github.com/sweinst)) -- Simon Mourier ([@smourier](https://github.com/smourier)) -- Viktoria Kovescses ([@vkovec](https://github.com/vkovec)) - Ville M. Vainio ([@vivainio](https://github.com/vivainio)) - Virgil Dupras ([@hsoft](https://github.com/hsoft)) - Wenguang Yang ([@yagweb](https://github.com/yagweb)) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3816cd0c..3dc668b0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,35 +8,23 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ## [unreleased][] ### Added - - Added support for embedding python into dotnet core 2.0 (NetStandard 2.0) - Added new build system (pythonnet.15.sln) based on dotnetcore-sdk/xplat(crossplatform msbuild). Currently there two side-by-side build systems that produces the same output (net40) from the same sources. After a some transition time, current (mono/ msbuild 14.0) build system will be removed. - NUnit upgraded to 3.7 (eliminates travis-ci random bug) -- Added C# `PythonEngine.AddShutdownHandler` to help client code clean up on shutdown. - Added `clr.GetClrType` ([#432][i432])([#433][p433]) - Allowed passing `None` for nullable args ([#460][p460]) - Added keyword arguments based on C# syntax for calling CPython methods ([#461][p461]) - Catches exceptions thrown in C# iterators (yield returns) and rethrows them in python ([#475][i475])([#693][p693]) - Implemented GetDynamicMemberNames() for PyObject to allow dynamic object members to be visible in the debugger ([#443][i443])([#690][p690]) - Incorporated reference-style links to issues and pull requests in the CHANGELOG ([#608][i608]) -- Added detailed comments about aproaches and dangers to handle multi-app-domains ([#625][p625]) -- Python 3.7 support, builds and testing added. Defaults changed from Python 3.6 to 3.7 ([#698][p698]) ### Changed -- Reattach python exception traceback information (#545) -- PythonEngine.Intialize will now call `Py_InitializeEx` with a default value of 0, so signals will not be configured by default on embedding. This is different from the previous behaviour, where `Py_Initialize` was called instead, which sets initSigs to 1. ([#449][i449]) - ### Fixed -- Fixed secondary PythonEngine.Initialize call, all sensitive static variables now reseted. - This is a hidden bug. Once python cleaning up enough memory, objects from previous engine run becomes corrupted. ([#534][p534]) - Fixed Visual Studio 2017 compat ([#434][i434]) for setup.py -- Fixed crashes when integrating pythonnet in Unity3d ([#714][i714]), - related to unloading the Application Domain -- Fixed interop methods with Py_ssize_t. NetCoreApp 2.0 is more sensitive than net40 and requires this fix. ([#531][p531]) - Fixed crash on exit of the Python interpreter if a python class derived from a .NET class has a `__namespace__` or `__assembly__` attribute ([#481][i481]) @@ -48,8 +36,6 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Fixed errors breaking .NET Remoting on method invoke ([#276][i276]) - Fixed PyObject.GetHashCode ([#676][i676]) - Fix memory leaks due to spurious handle incrementation ([#691][i691]) -- Fix spurious assembly loading exceptions from private types ([#703][i703]) -- Fix inheritance of non-abstract base methods ([#755][i755]) ## [2.3.0][] - 2017-03-11 @@ -607,7 +593,6 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. [1.0.0]: https://github.com/pythonnet/pythonnet/releases/tag/1.0 -[i714]: https://github.com/pythonnet/pythonnet/issues/714 [i608]: https://github.com/pythonnet/pythonnet/issues/608 [i443]: https://github.com/pythonnet/pythonnet/issues/443 [p690]: https://github.com/pythonnet/pythonnet/pull/690 @@ -698,9 +683,4 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. [p225]: https://github.com/pythonnet/pythonnet/pull/225 [p78]: https://github.com/pythonnet/pythonnet/pull/78 [p163]: https://github.com/pythonnet/pythonnet/pull/163 -[p625]: https://github.com/pythonnet/pythonnet/pull/625 [i131]: https://github.com/pythonnet/pythonnet/issues/131 -[p531]: https://github.com/pythonnet/pythonnet/pull/531 -[i755]: https://github.com/pythonnet/pythonnet/pull/755 -[p534]: https://github.com/pythonnet/pythonnet/pull/534 -[i449]: https://github.com/pythonnet/pythonnet/issues/449 diff --git a/NuGet.config b/NuGet.config index 5210cd6c9..719fbc83c 100644 --- a/NuGet.config +++ b/NuGet.config @@ -1,4 +1,4 @@ - + diff --git a/appveyor.yml b/appveyor.yml index ce345cbf8..6bebef490 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,19 +23,10 @@ environment: BUILD_OPTS: --xplat - PYTHON_VERSION: 3.6 BUILD_OPTS: --xplat - - PYTHON_VERSION: 3.7 - BUILD_OPTS: --xplat - PYTHON_VERSION: 2.7 - PYTHON_VERSION: 3.4 - PYTHON_VERSION: 3.5 - PYTHON_VERSION: 3.6 - - PYTHON_VERSION: 3.7 - -matrix: - allow_failures: - - PYTHON_VERSION: 3.4 - BUILD_OPTS: --xplat - - PYTHON_VERSION: 3.4 init: # Update Environment Variables based on matrix/platform diff --git a/setup.py b/setup.py index 10f1b178b..3695e33fa 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ # Allow config/verbosity to be set from cli # http://stackoverflow.com/a/4792601/5208670 CONFIG = "Release" # Release or Debug -VERBOSITY = "normal" # quiet, minimal, normal, detailed, diagnostic +VERBOSITY = "minimal" # quiet, minimal, normal, detailed, diagnostic is_64bits = sys.maxsize > 2**32 DEVTOOLS = "MsDev" if sys.platform == "win32" else "Mono" @@ -329,16 +329,9 @@ def _install_packages(self): self.debug_print("Updating NuGet: {0}".format(cmd)) subprocess.check_call(cmd, shell=use_shell) - try: - # msbuild=14 is mainly for Mono issues - cmd = "{0} restore pythonnet.sln -MSBuildVersion 14 -o packages".format(nuget) - self.debug_print("Installing packages: {0}".format(cmd)) - subprocess.check_call(cmd, shell=use_shell) - except: - # when only VS 2017 is installed do not specify msbuild version - cmd = "{0} restore pythonnet.sln -o packages".format(nuget) - self.debug_print("Installing packages: {0}".format(cmd)) - subprocess.check_call(cmd, shell=use_shell) + cmd = "{0} restore pythonnet.sln -o packages".format(nuget) + self.debug_print("Installing packages: {0}".format(cmd)) + subprocess.check_call(cmd, shell=use_shell) def _find_msbuild_tool(self, tool="msbuild.exe", use_windows_sdk=False): """Return full path to one of the Microsoft build tools""" @@ -523,10 +516,10 @@ def run(self): 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Operating System :: Microsoft :: Windows', 'Operating System :: POSIX :: Linux', 'Operating System :: MacOS :: MacOS X', diff --git a/src/embed_tests/GlobalTestsSetup.cs b/src/embed_tests/GlobalTestsSetup.cs deleted file mode 100644 index 458ab6a99..000000000 --- a/src/embed_tests/GlobalTestsSetup.cs +++ /dev/null @@ -1,21 +0,0 @@ -using NUnit.Framework; -using Python.Runtime; - -namespace Python.EmbeddingTest -{ - - // As the SetUpFixture, the OneTimeTearDown of this class is executed after - // all tests have run. - [SetUpFixture] - public class GlobalTestsSetup - { - [OneTimeTearDown] - public void FinalCleanup() - { - if (PythonEngine.IsInitialized) - { - PythonEngine.Shutdown(); - } - } - } -} diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj index a741a589e..92d55a7e0 100644 --- a/src/embed_tests/Python.EmbeddingTest.15.csproj +++ b/src/embed_tests/Python.EmbeddingTest.15.csproj @@ -29,7 +29,6 @@ XPLAT $(DefineConstants);$(CustomDefineConstants);$(BaseDefineConstants); $(DefineConstants);NETCOREAPP - $(DefineConstants);NETSTANDARD $(DefineConstants);TRACE;DEBUG $(NuGetPackageRoot)\microsoft.targetingpack.netframework.v4.5\1.0.1\lib\net45\ diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index 6aa48becc..66e8c7165 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -1,4 +1,4 @@ - + Debug @@ -86,7 +86,6 @@ - @@ -104,8 +103,6 @@ - - @@ -125,4 +122,4 @@ - + \ No newline at end of file diff --git a/src/embed_tests/TestConverter.cs b/src/embed_tests/TestConverter.cs index caaec311b..346c8afdc 100644 --- a/src/embed_tests/TestConverter.cs +++ b/src/embed_tests/TestConverter.cs @@ -1,4 +1,4 @@ -using NUnit.Framework; +using NUnit.Framework; using Python.Runtime; namespace Python.EmbeddingTest diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs deleted file mode 100644 index b162d4eb0..000000000 --- a/src/embed_tests/TestDomainReload.cs +++ /dev/null @@ -1,239 +0,0 @@ -using System; -using System.CodeDom.Compiler; -using System.Reflection; -using NUnit.Framework; -using Python.Runtime; - -// -// This test case is disabled on .NET Standard because it doesn't have all the -// APIs we use. We could work around that, but .NET Core doesn't implement -// domain creation, so it's not worth it. -// -// Unfortunately this means no continuous integration testing for this case. -// -#if !NETSTANDARD && !NETCOREAPP -namespace Python.EmbeddingTest -{ - class TestDomainReload - { - /// - /// Test that the python runtime can survive a C# domain reload without crashing. - /// - /// At the time this test was written, there was a very annoying - /// seemingly random crash bug when integrating pythonnet into Unity. - /// - /// The repro steps that David Lassonde, Viktoria Kovecses and - /// Benoit Hudson eventually worked out: - /// 1. Write a HelloWorld.cs script that uses Python.Runtime to access - /// some C# data from python: C# calls python, which calls C#. - /// 2. Execute the script (e.g. make it a MenuItem and click it). - /// 3. Touch HelloWorld.cs on disk, forcing Unity to recompile scripts. - /// 4. Wait several seconds for Unity to be done recompiling and - /// reloading the C# domain. - /// 5. Make python run the gc (e.g. by calling gc.collect()). - /// - /// The reason: - /// A. In step 2, Python.Runtime registers a bunch of new types with - /// their tp_traverse slot pointing to managed code, and allocates - /// some objects of those types. - /// B. In step 4, Unity unloads the C# domain. That frees the managed - /// code. But at the time of the crash investigation, pythonnet - /// leaked the python side of the objects allocated in step 1. - /// C. In step 5, python sees some pythonnet objects in its gc list of - /// potentially-leaked objects. It calls tp_traverse on those objects. - /// But tp_traverse was freed in step 3 => CRASH. - /// - /// This test distills what's going on without needing Unity around (we'd see - /// similar behaviour if we were using pythonnet on a .NET web server that did - /// a hot reload). - /// - [Test] - public static void DomainReloadAndGC() - { - // We're set up to run in the directory that includes the bin directory. - System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); - - Assembly pythonRunner1 = BuildAssembly("test1"); - RunAssemblyAndUnload(pythonRunner1, "test1"); - - // Verify that python is not initialized even though we ran it. - Assert.That(Runtime.Runtime.Py_IsInitialized(), Is.Zero); - - // This caused a crash because objects allocated in pythonRunner1 - // still existed in memory, but the code to do python GC on those - // objects is gone. - Assembly pythonRunner2 = BuildAssembly("test2"); - RunAssemblyAndUnload(pythonRunner2, "test2"); - } - - // - // The code we'll test. All that really matters is - // using GIL { Python.Exec(pyScript); } - // but the rest is useful for debugging. - // - // What matters in the python code is gc.collect and clr.AddReference. - // - // Note that the language version is 2.0, so no $"foo{bar}" syntax. - // - const string TestCode = @" - using Python.Runtime; - using System; - class PythonRunner { - public static void RunPython() { - AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; - string name = AppDomain.CurrentDomain.FriendlyName; - Console.WriteLine(string.Format(""[{0} in .NET] In PythonRunner.RunPython"", name)); - using (Py.GIL()) { - try { - var pyScript = string.Format(""import clr\n"" - + ""print('[{0} in python] imported clr')\n"" - + ""clr.AddReference('System')\n"" - + ""print('[{0} in python] allocated a clr object')\n"" - + ""import gc\n"" - + ""gc.collect()\n"" - + ""print('[{0} in python] collected garbage')\n"", - name); - PythonEngine.Exec(pyScript); - } catch(Exception e) { - Console.WriteLine(string.Format(""[{0} in .NET] Caught exception: {1}"", name, e)); - } - } - } - static void OnDomainUnload(object sender, EventArgs e) { - System.Console.WriteLine(string.Format(""[{0} in .NET] unloading"", AppDomain.CurrentDomain.FriendlyName)); - } - }"; - - - /// - /// Build an assembly out of the source code above. - /// - /// This creates a file .dll in order - /// to support the statement "proxy.theAssembly = assembly" below. - /// That statement needs a file, can't run via memory. - /// - static Assembly BuildAssembly(string assemblyName) - { - var provider = CodeDomProvider.CreateProvider("CSharp"); - - var compilerparams = new CompilerParameters(); - compilerparams.ReferencedAssemblies.Add("Python.Runtime.dll"); - compilerparams.GenerateExecutable = false; - compilerparams.GenerateInMemory = false; - compilerparams.IncludeDebugInformation = false; - compilerparams.OutputAssembly = assemblyName; - - var results = provider.CompileAssemblyFromSource(compilerparams, TestCode); - if (results.Errors.HasErrors) - { - var errors = new System.Text.StringBuilder("Compiler Errors:\n"); - foreach (CompilerError error in results.Errors) - { - errors.AppendFormat("Line {0},{1}\t: {2}\n", - error.Line, error.Column, error.ErrorText); - } - throw new Exception(errors.ToString()); - } - else - { - return results.CompiledAssembly; - } - } - - /// - /// This is a magic incantation required to run code in an application - /// domain other than the current one. - /// - class Proxy : MarshalByRefObject - { - Assembly theAssembly = null; - - public void InitAssembly(string assemblyPath) - { - theAssembly = Assembly.LoadFile(System.IO.Path.GetFullPath(assemblyPath)); - } - - public void RunPython() - { - Console.WriteLine("[Proxy] Entering RunPython"); - - // Call into the new assembly. Will execute Python code - var pythonrunner = theAssembly.GetType("PythonRunner"); - var runPythonMethod = pythonrunner.GetMethod("RunPython"); - runPythonMethod.Invoke(null, new object[] { }); - - Console.WriteLine("[Proxy] Leaving RunPython"); - } - } - - /// - /// Create a domain, run the assembly in it (the RunPython function), - /// and unload the domain. - /// - static void RunAssemblyAndUnload(Assembly assembly, string assemblyName) - { - Console.WriteLine($"[Program.Main] === creating domain for assembly {assembly.FullName}"); - - // Create the domain. Make sure to set PrivateBinPath to a relative - // path from the CWD (namely, 'bin'). - // See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain - var currentDomain = AppDomain.CurrentDomain; - var domainsetup = new AppDomainSetup() - { - ApplicationBase = currentDomain.SetupInformation.ApplicationBase, - ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, - LoaderOptimization = LoaderOptimization.SingleDomain, - PrivateBinPath = "." - }; - var domain = AppDomain.CreateDomain( - $"My Domain {assemblyName}", - currentDomain.Evidence, - domainsetup); - - // Create a Proxy object in the new domain, where we want the - // assembly (and Python .NET) to reside - Type type = typeof(Proxy); - System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); - var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( - type.Assembly.FullName, - type.FullName); - - // From now on use the Proxy to call into the new assembly - theProxy.InitAssembly(assemblyName); - theProxy.RunPython(); - - Console.WriteLine($"[Program.Main] Before Domain Unload on {assembly.FullName}"); - AppDomain.Unload(domain); - Console.WriteLine($"[Program.Main] After Domain Unload on {assembly.FullName}"); - - // Validate that the assembly does not exist anymore - try - { - Console.WriteLine($"[Program.Main] The Proxy object is valid ({theProxy}). Unexpected domain unload behavior"); - } - catch (Exception) - { - Console.WriteLine("[Program.Main] The Proxy object is not valid anymore, domain unload complete."); - } - } - - /// - /// Resolves the assembly. Why doesn't this just work normally? - /// - static Assembly ResolveAssembly(object sender, ResolveEventArgs args) - { - var loadedAssemblies = AppDomain.CurrentDomain.GetAssemblies(); - - foreach (var assembly in loadedAssemblies) - { - if (assembly.FullName == args.Name) - { - return assembly; - } - } - - return null; - } - } -} -#endif diff --git a/src/embed_tests/TestNamedArguments.cs b/src/embed_tests/TestNamedArguments.cs index 31f2ea1d2..1d7076956 100644 --- a/src/embed_tests/TestNamedArguments.cs +++ b/src/embed_tests/TestNamedArguments.cs @@ -1,4 +1,4 @@ -using System; +using System; using NUnit.Framework; using Python.Runtime; diff --git a/src/embed_tests/TestPyObject.cs b/src/embed_tests/TestPyObject.cs index 65ac20e9a..d794ce06e 100644 --- a/src/embed_tests/TestPyObject.cs +++ b/src/embed_tests/TestPyObject.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using NUnit.Framework; diff --git a/src/embed_tests/TestPySequence.cs b/src/embed_tests/TestPySequence.cs index 7c175b1ce..1e3ebf144 100644 --- a/src/embed_tests/TestPySequence.cs +++ b/src/embed_tests/TestPySequence.cs @@ -69,8 +69,10 @@ public void TestRepeat() PyObject actual = t1.Repeat(3); Assert.AreEqual("FooFooFoo", actual.ToString()); - actual = t1.Repeat(-3); - Assert.AreEqual("", actual.ToString()); + // On 32 bit system this argument should be int, but on the 64 bit system this should be long value. + // This works on the Framework 4.0 accidentally, it should produce out of memory! + // actual = t1.Repeat(-3); + // Assert.AreEqual("", actual.ToString()); } [Test] diff --git a/src/embed_tests/TestPyWith.cs b/src/embed_tests/TestPyWith.cs index 0c4e9023f..fd3f8e662 100644 --- a/src/embed_tests/TestPyWith.cs +++ b/src/embed_tests/TestPyWith.cs @@ -1,4 +1,4 @@ -using System; +using System; using NUnit.Framework; using Python.Runtime; diff --git a/src/embed_tests/TestPythonEngineProperties.cs b/src/embed_tests/TestPythonEngineProperties.cs index 243349b82..01c6ae7e3 100644 --- a/src/embed_tests/TestPythonEngineProperties.cs +++ b/src/embed_tests/TestPythonEngineProperties.cs @@ -109,41 +109,18 @@ public static void GetPythonHomeDefault() [Test] public void SetPythonHome() { - // We needs to ensure that engine was started and shutdown at least once before setting dummy home. - // Otherwise engine will not run with dummy path with random problem. - if (!PythonEngine.IsInitialized) - { - PythonEngine.Initialize(); - } - - PythonEngine.Shutdown(); - - var pythonHomeBackup = PythonEngine.PythonHome; - var pythonHome = "/dummypath/"; PythonEngine.PythonHome = pythonHome; PythonEngine.Initialize(); + Assert.AreEqual(pythonHome, PythonEngine.PythonHome); PythonEngine.Shutdown(); - - // Restoring valid pythonhome. - PythonEngine.PythonHome = pythonHomeBackup; } [Test] public void SetPythonHomeTwice() { - // We needs to ensure that engine was started and shutdown at least once before setting dummy home. - // Otherwise engine will not run with dummy path with random problem. - if (!PythonEngine.IsInitialized) - { - PythonEngine.Initialize(); - } - PythonEngine.Shutdown(); - - var pythonHomeBackup = PythonEngine.PythonHome; - var pythonHome = "/dummypath/"; PythonEngine.PythonHome = "/dummypath2/"; @@ -152,20 +129,11 @@ public void SetPythonHomeTwice() Assert.AreEqual(pythonHome, PythonEngine.PythonHome); PythonEngine.Shutdown(); - - PythonEngine.PythonHome = pythonHomeBackup; } [Test] public void SetProgramName() { - if (PythonEngine.IsInitialized) - { - PythonEngine.Shutdown(); - } - - var programNameBackup = PythonEngine.ProgramName; - var programName = "FooBar"; PythonEngine.ProgramName = programName; @@ -173,8 +141,6 @@ public void SetProgramName() Assert.AreEqual(programName, PythonEngine.ProgramName); PythonEngine.Shutdown(); - - PythonEngine.ProgramName = programNameBackup; } [Test] @@ -190,7 +156,7 @@ public void SetPythonPath() string path = PythonEngine.PythonPath; PythonEngine.Shutdown(); - PythonEngine.PythonPath = path; + PythonEngine.ProgramName = path; PythonEngine.Initialize(); Assert.AreEqual(path, PythonEngine.PythonPath); @@ -205,6 +171,7 @@ public void SetPythonPathExceptionOn27() Assert.Pass(); } + // Get previous path to avoid crashing Python PythonEngine.Initialize(); string path = PythonEngine.PythonPath; PythonEngine.Shutdown(); diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index ac1fa1ac0..2e0598da7 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -1,4 +1,4 @@ -using System; +using System; using NUnit.Framework; using Python.Runtime; @@ -6,40 +6,10 @@ namespace Python.EmbeddingTest { public class TestRuntime { - [OneTimeSetUp] - public void SetUp() - { - // We needs to ensure that no any engines are running. - if (PythonEngine.IsInitialized) - { - PythonEngine.Shutdown(); - } - } - - /// - /// Test the cache of the information from the platform module. - /// - /// Test fails on platforms we haven't implemented yet. - /// - [Test] - public static void PlatformCache() - { - Runtime.Runtime.Initialize(); - - Assert.That(Runtime.Runtime.Machine, Is.Not.EqualTo(Runtime.Runtime.MachineType.Other)); - Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.MachineName)); - - Assert.That(Runtime.Runtime.OperatingSystem, Is.Not.EqualTo(Runtime.Runtime.OperatingSystemType.Other)); - Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.OperatingSystemName)); - - // Don't shut down the runtime: if the python engine was initialized - // but not shut down by another test, we'd end up in a bad state. - } - [Test] public static void Py_IsInitializedValue() { - Runtime.Runtime.Py_Finalize(); + Runtime.Runtime.Py_Finalize(); // In case another test left it on. Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); Runtime.Runtime.Py_Initialize(); Assert.AreEqual(1, Runtime.Runtime.Py_IsInitialized()); diff --git a/src/embed_tests/TestTypeManager.cs b/src/embed_tests/TestTypeManager.cs deleted file mode 100644 index a4ef86913..000000000 --- a/src/embed_tests/TestTypeManager.cs +++ /dev/null @@ -1,65 +0,0 @@ -using NUnit.Framework; -using Python.Runtime; -using System.Runtime.InteropServices; - -namespace Python.EmbeddingTest -{ - class TestTypeManager - { - [SetUp] - public static void Init() - { - Runtime.Runtime.Initialize(); - } - - [TearDown] - public static void Fini() - { - // Don't shut down the runtime: if the python engine was initialized - // but not shut down by another test, we'd end up in a bad state. - } - - [Test] - public static void TestNativeCode() - { - Assert.That(() => { var _ = TypeManager.NativeCode.Active; }, Throws.Nothing); - Assert.That(TypeManager.NativeCode.Active.Code.Length, Is.GreaterThan(0)); - } - - [Test] - public static void TestMemoryMapping() - { - Assert.That(() => { var _ = TypeManager.CreateMemoryMapper(); }, Throws.Nothing); - var mapper = TypeManager.CreateMemoryMapper(); - - // Allocate a read-write page. - int len = 12; - var page = mapper.MapWriteable(len); - Assert.That(() => { Marshal.WriteInt64(page, 17); }, Throws.Nothing); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - - // Mark it read-execute. We can still read, haven't changed any values. - mapper.SetReadExec(page, len); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - - // Test that we can't write to the protected page. - // - // We can't actually test access protection under Microsoft - // versions of .NET, because AccessViolationException is assumed to - // mean we're in a corrupted state: - // https://stackoverflow.com/questions/3469368/how-to-handle-accessviolationexception - // - // We can test under Mono but it throws NRE instead of AccessViolationException. - // - // We can't use compiler flags because we compile with MONO_LINUX - // while running on the Microsoft .NET Core during continuous - // integration tests. - if (System.Type.GetType ("Mono.Runtime") != null) - { - // Mono throws NRE instead of AccessViolationException for some reason. - Assert.That(() => { Marshal.WriteInt64(page, 73); }, Throws.TypeOf()); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - } - } - } -} diff --git a/src/embed_tests/dynamic.cs b/src/embed_tests/dynamic.cs index 81345cee7..d75dc01d6 100644 --- a/src/embed_tests/dynamic.cs +++ b/src/embed_tests/dynamic.cs @@ -12,23 +12,13 @@ public class DynamicTest [SetUp] public void SetUp() { - try { - _gs = Py.GIL(); - } catch (Exception e) { - Console.WriteLine($"exception in SetUp: {e}"); - throw; - } + _gs = Py.GIL(); } [TearDown] public void Dispose() { - try { - _gs.Dispose(); - } catch(Exception e) { - Console.WriteLine($"exception in TearDown: {e}"); - throw; - } + _gs.Dispose(); } /// @@ -128,11 +118,10 @@ public void PassPyObjectInNet() sys.testattr2 = sys.testattr1; // Compare in Python - PythonEngine.RunSimpleString( + PyObject res = PythonEngine.RunString( "import sys\n" + "sys.testattr3 = sys.testattr1 is sys.testattr2\n" ); - Assert.AreEqual(sys.testattr3.ToString(), "True"); // Compare in .NET diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs index ea1d8d023..2f9aae2c7 100644 --- a/src/embed_tests/pyinitialize.cs +++ b/src/embed_tests/pyinitialize.cs @@ -74,65 +74,5 @@ public void ReInitialize() } PythonEngine.Shutdown(); } - - [Test] - public void TestScopeIsShutdown() - { - PythonEngine.Initialize(); - var scope = PyScopeManager.Global.Create("test"); - PythonEngine.Shutdown(); - Assert.That(PyScopeManager.Global.Contains("test"), Is.False); - } - - /// - /// Helper for testing the shutdown handlers. - /// - int shutdown_count = 0; - void OnShutdownIncrement() - { - shutdown_count++; - } - void OnShutdownDouble() - { - shutdown_count *= 2; - } - - /// - /// Test the shutdown handlers. - /// - [Test] - public void ShutdownHandlers() - { - // Test we can run one shutdown handler. - shutdown_count = 0; - PythonEngine.Initialize(); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.Shutdown(); - Assert.That(shutdown_count, Is.EqualTo(1)); - - // Test we can run multiple shutdown handlers in the right order. - shutdown_count = 4; - PythonEngine.Initialize(); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.AddShutdownHandler(OnShutdownDouble); - PythonEngine.Shutdown(); - // Correct: 4 * 2 + 1 = 9 - // Wrong: (4 + 1) * 2 = 10 - Assert.That(shutdown_count, Is.EqualTo(9)); - - // Test we can remove shutdown handlers, handling duplicates. - shutdown_count = 4; - PythonEngine.Initialize(); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.AddShutdownHandler(OnShutdownDouble); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.AddShutdownHandler(OnShutdownDouble); - PythonEngine.RemoveShutdownHandler(OnShutdownDouble); - PythonEngine.Shutdown(); - // Correct: (4 + 1) * 2 + 1 + 1 = 12 - // Wrong: (4 * 2) + 1 + 1 + 1 = 11 - Assert.That(shutdown_count, Is.EqualTo(12)); - } } } diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj index 794645994..cfde0a127 100644 --- a/src/runtime/Python.Runtime.15.csproj +++ b/src/runtime/Python.Runtime.15.csproj @@ -35,7 +35,7 @@ $(PYTHONNET_PY2_VERSION) PYTHON27 $(PYTHONNET_PY3_VERSION) - PYTHON37 + PYTHON36 $(PYTHONNET_WIN_DEFINE_CONSTANTS) UCS2 $(PYTHONNET_MONO_DEFINE_CONSTANTS) @@ -131,7 +131,7 @@ - + diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index fc155ca91..1fea78082 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,4 +1,4 @@ - + Debug @@ -34,7 +34,7 @@ pdbonly - PYTHON3;PYTHON37;UCS4 + PYTHON3;PYTHON36;UCS4 true pdbonly @@ -46,7 +46,7 @@ true - PYTHON3;PYTHON37;UCS4;TRACE;DEBUG + PYTHON3;PYTHON36;UCS4;TRACE;DEBUG false full @@ -56,7 +56,7 @@ pdbonly - PYTHON3;PYTHON37;UCS2 + PYTHON3;PYTHON36;UCS2 true pdbonly @@ -68,7 +68,7 @@ true - PYTHON3;PYTHON37;UCS2;TRACE;DEBUG + PYTHON3;PYTHON36;UCS2;TRACE;DEBUG false full @@ -148,7 +148,6 @@ - @@ -167,4 +166,4 @@ - + \ No newline at end of file diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs index dc5f78608..dd4418cc9 100644 --- a/src/runtime/Util.cs +++ b/src/runtime/Util.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Runtime.InteropServices; namespace Python.Runtime diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index c37295704..a10688749 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -93,7 +93,7 @@ public static IntPtr mp_subscript(IntPtr ob, IntPtr idx) return IntPtr.Zero; } - var count = Runtime.PyTuple_Size(idx); + int count = Runtime.PyTuple_Size(idx); var args = new int[count]; @@ -186,7 +186,7 @@ public static int mp_ass_subscript(IntPtr ob, IntPtr idx, IntPtr v) return -1; } - var count = Runtime.PyTuple_Size(idx); + int count = Runtime.PyTuple_Size(idx); var args = new int[count]; for (var i = 0; i < count; i++) diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 3085bb639..d63930a58 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Linq; using System.Reflection; using System.Threading; @@ -18,21 +17,13 @@ internal class AssemblyManager { // modified from event handlers below, potentially triggered from different .NET threads // therefore this should be a ConcurrentDictionary - // - // WARNING: Dangerous if cross-app domain usage is ever supported - // Reusing the dictionary with assemblies accross multiple initializations is problematic. - // Loading happens from CurrentDomain (see line 53). And if the first call is from AppDomain that is later unloaded, - // than it can end up referring to assemblies that are already unloaded (default behavior after unload appDomain - - // unless LoaderOptimization.MultiDomain is used); - // So for multidomain support it is better to have the dict. recreated for each app-domain initialization - private static ConcurrentDictionary> namespaces = - new ConcurrentDictionary>(); + private static ConcurrentDictionary> namespaces; //private static Dictionary> generics; private static AssemblyLoadEventHandler lhandler; private static ResolveEventHandler rhandler; // updated only under GIL? - private static Dictionary probed = new Dictionary(32); + private static Dictionary probed; // modified from event handlers below, potentially triggered from different .NET threads private static ConcurrentQueue assemblies; @@ -49,6 +40,9 @@ private AssemblyManager() /// internal static void Initialize() { + namespaces = new ConcurrentDictionary>(); + probed = new Dictionary(32); + //generics = new Dictionary>(); assemblies = new ConcurrentQueue(); pypath = new List(16); @@ -138,7 +132,7 @@ private static Assembly ResolveHandler(object ob, ResolveEventArgs args) internal static void UpdatePath() { IntPtr list = Runtime.PySys_GetObject("path"); - var count = Runtime.PyList_Size(list); + int count = Runtime.PyList_Size(list); if (count != pypath.Count) { pypath.Clear(); @@ -349,7 +343,9 @@ internal static void ScanAssembly(Assembly assembly) // A couple of things we want to do here: first, we want to // gather a list of all of the namespaces contributed to by // the assembly. - foreach (Type t in GetTypes(assembly)) + + Type[] types = assembly.GetTypes(); + foreach (Type t in types) { string ns = t.Namespace ?? ""; if (!namespaces.ContainsKey(ns)) @@ -423,9 +419,10 @@ public static List GetNames(string nsname) { foreach (Assembly a in namespaces[nsname].Keys) { - foreach (Type t in GetTypes(a)) + Type[] types = a.GetTypes(); + foreach (Type t in types) { - if ((t.Namespace ?? "") == nsname && !t.IsNested) + if ((t.Namespace ?? "") == nsname) { names.Add(t.Name); } @@ -464,32 +461,5 @@ public static Type LookupType(string qname) } return null; } - - internal static Type[] GetTypes(Assembly a) - { - if (a.IsDynamic) - { - try - { - return a.GetTypes(); - } - catch (ReflectionTypeLoadException exc) - { - // Return all types that were successfully loaded - return exc.Types.Where(x => x != null).ToArray(); - } - } - else - { - try - { - return a.GetExportedTypes(); - } - catch (FileNotFoundException) - { - return new Type[0]; - } - } - } } -} \ No newline at end of file +} diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 5846fa82a..4dd3b5364 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -247,6 +247,24 @@ public static IntPtr tp_str(IntPtr ob) } + /// + /// Default implementations for required Python GC support. + /// + public static int tp_traverse(IntPtr ob, IntPtr func, IntPtr args) + { + return 0; + } + + public static int tp_clear(IntPtr ob) + { + return 0; + } + + public static int tp_is_gc(IntPtr type) + { + return 1; + } + /// /// Standard dealloc implementation for instances of reflected types. /// diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index ec3734ea5..16d3b99db 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -1,9 +1,8 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Reflection; using System.Reflection.Emit; -using System.Resources; using System.Runtime.InteropServices; using System.Threading.Tasks; @@ -33,12 +32,6 @@ static ClassDerivedObject() moduleBuilders = new Dictionary, ModuleBuilder>(); } - public static void Reset() - { - assemblyBuilders = new Dictionary(); - moduleBuilders = new Dictionary, ModuleBuilder>(); - } - internal ClassDerivedObject(Type tp) : base(tp) { } diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 0b084a49d..6a9d40ebd 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -34,11 +34,6 @@ static ClassManager() dtype = typeof(MulticastDelegate); } - public static void Reset() - { - cache = new Dictionary(128); - } - /// /// Return the ClassBase-derived instance that implements a particular /// reflected managed type, creating it if it doesn't yet exist. diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs index 83d761fd0..46257c73f 100644 --- a/src/runtime/classobject.cs +++ b/src/runtime/classobject.cs @@ -230,10 +230,10 @@ public static int mp_ass_subscript(IntPtr ob, IntPtr idx, IntPtr v) } // Get the args passed in. - var i = Runtime.PyTuple_Size(args); + int i = Runtime.PyTuple_Size(args); IntPtr defaultArgs = cls.indexer.GetDefaultArgs(args); - var numOfDefaultArgs = Runtime.PyTuple_Size(defaultArgs); - var temp = i + numOfDefaultArgs; + int numOfDefaultArgs = Runtime.PyTuple_Size(defaultArgs); + int temp = i + numOfDefaultArgs; IntPtr real = Runtime.PyTuple_New(temp + 1); for (var n = 0; n < i; n++) { diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 502677655..fb3d0e0d7 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Runtime.InteropServices; namespace Python.Runtime diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 629eab4df..d8ec3fcd2 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -3,7 +3,6 @@ using System.Globalization; using System.Runtime.InteropServices; using System.Security; -using System.ComponentModel; namespace Python.Runtime { @@ -167,8 +166,8 @@ internal static IntPtr ToPython(object value, Type type) return result; } - if (value is IList && !(value is INotifyPropertyChanged) && value.GetType().IsGenericType) - { + if (value is IList && value.GetType().IsGenericType) + { using (var resultlist = new PyList()) { foreach (object o in (IEnumerable)value) @@ -1002,7 +1001,7 @@ private static void SetConversionError(IntPtr value, Type target) private static bool ToArray(IntPtr value, Type obType, out object result, bool setError) { Type elementType = obType.GetElementType(); - var size = Runtime.PySequence_Size(value); + int size = Runtime.PySequence_Size(value); result = null; if (size < 0 || elementType.IsGenericType) diff --git a/src/runtime/debughelper.cs b/src/runtime/debughelper.cs index 3fe9ee5bb..2a91a74b4 100644 --- a/src/runtime/debughelper.cs +++ b/src/runtime/debughelper.cs @@ -116,10 +116,10 @@ internal static void debug(string msg) Console.WriteLine(" {0}", msg); } - /// + /// /// Helper function to inspect/compare managed to native conversions. - /// Especially useful when debugging CustomMarshaler. - /// + /// Especially useful when debugging CustomMarshaler. + /// /// [Conditional("DEBUG")] public static void PrintHexBytes(byte[] bytes) diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 8bed0abfd..9023cfcfa 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -256,10 +256,7 @@ public static void SetError(Exception e) var pe = e as PythonException; if (pe != null) { - Runtime.XIncref(pe.PyType); - Runtime.XIncref(pe.PyValue); - Runtime.XIncref(pe.PyTB); - Runtime.PyErr_Restore(pe.PyType, pe.PyValue, pe.PyTB); + Runtime.PyErr_SetObject(pe.PyType, pe.PyValue); return; } @@ -279,7 +276,7 @@ public static void SetError(Exception e) /// public static bool ErrorOccurred() { - return Runtime.PyErr_Occurred() != IntPtr.Zero; + return Runtime.PyErr_Occurred() != 0; } /// diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 693a46f42..9569b0485 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -81,6 +81,27 @@ public static int tp_descr_set(IntPtr ds, IntPtr ob, IntPtr val) } + /// + /// Required Python GC support. + /// + public static int tp_traverse(IntPtr ob, IntPtr func, IntPtr args) + { + return 0; + } + + + public static int tp_clear(IntPtr ob) + { + return 0; + } + + + public static int tp_is_gc(IntPtr type) + { + return 1; + } + + /// /// Default dealloc implementation. /// diff --git a/src/runtime/genericutil.cs b/src/runtime/genericutil.cs index 3a230e12c..9772d082f 100644 --- a/src/runtime/genericutil.cs +++ b/src/runtime/genericutil.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Resources; namespace Python.Runtime { @@ -21,11 +20,6 @@ static GenericUtil() mapping = new Dictionary>>(); } - public static void Reset() - { - mapping = new Dictionary>>(); - } - /// /// Register a generic type that appears in a given namespace. /// diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 7e4a208f5..bc9ac5eee 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -155,7 +155,7 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) // hook is saved as this.py_import. This version handles CLR // import and defers to the normal builtin for everything else. - var num_args = Runtime.PyTuple_Size(args); + int num_args = Runtime.PyTuple_Size(args); if (num_args < 1) { return Exceptions.RaiseTypeError("__import__() takes at least 1 argument (0 given)"); @@ -237,11 +237,6 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) if (res != IntPtr.Zero) { // There was no error. - if (fromlist && IsLoadAll(fromList)) - { - var mod = ManagedType.GetManagedObject(res) as ModuleObject; - mod?.LoadNames(); - } return res; } // There was an error @@ -295,11 +290,6 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) { if (fromlist) { - if (IsLoadAll(fromList)) - { - var mod = ManagedType.GetManagedObject(module) as ModuleObject; - mod?.LoadNames(); - } Runtime.XIncref(module); return module; } @@ -355,33 +345,20 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) } } - { - var mod = fromlist ? tail : head; + ModuleObject mod = fromlist ? tail : head; - if (fromlist && IsLoadAll(fromList)) + if (fromlist && Runtime.PySequence_Size(fromList) == 1) + { + IntPtr fp = Runtime.PySequence_GetItem(fromList, 0); + if (!CLRModule.preload && Runtime.GetManagedString(fp) == "*") { mod.LoadNames(); } - - Runtime.XIncref(mod.pyHandle); - return mod.pyHandle; + Runtime.XDecref(fp); } - } - private static bool IsLoadAll(IntPtr fromList) - { - if (CLRModule.preload) - { - return false; - } - if (Runtime.PySequence_Size(fromList) != 1) - { - return false; - } - IntPtr fp = Runtime.PySequence_GetItem(fromList, 0); - bool res = Runtime.GetManagedString(fp) == "*"; - Runtime.XDecref(fp); - return res; + Runtime.XIncref(mod.pyHandle); + return mod.pyHandle; } } } diff --git a/src/runtime/indexer.cs b/src/runtime/indexer.cs index 71f7e7aa1..7b6d90ca8 100644 --- a/src/runtime/indexer.cs +++ b/src/runtime/indexer.cs @@ -56,7 +56,7 @@ internal void SetItem(IntPtr inst, IntPtr args) internal bool NeedsDefaultArgs(IntPtr args) { - var pynargs = Runtime.PyTuple_Size(args); + int pynargs = Runtime.PyTuple_Size(args); MethodBase[] methods = SetterBinder.GetMethods(); if (methods.Length == 0) { @@ -72,7 +72,7 @@ internal bool NeedsDefaultArgs(IntPtr args) return false; } - for (var v = pynargs; v < clrnargs; v++) + for (int v = pynargs; v < clrnargs; v++) { if (pi[v].DefaultValue == DBNull.Value) { @@ -95,7 +95,7 @@ internal IntPtr GetDefaultArgs(IntPtr args) { return Runtime.PyTuple_New(0); } - var pynargs = Runtime.PyTuple_Size(args); + int pynargs = Runtime.PyTuple_Size(args); // Get the default arg tuple MethodBase[] methods = SetterBinder.GetMethods(); diff --git a/src/runtime/interfaceobject.cs b/src/runtime/interfaceobject.cs index 616ced6bd..ce1bc9eb0 100644 --- a/src/runtime/interfaceobject.cs +++ b/src/runtime/interfaceobject.cs @@ -36,7 +36,7 @@ static InterfaceObject() public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { var self = (InterfaceObject)GetManagedObject(tp); - var nargs = Runtime.PyTuple_Size(args); + int nargs = Runtime.PyTuple_Size(args); Type type = self.type; object obj; diff --git a/src/runtime/interop37.cs b/src/runtime/interop37.cs deleted file mode 100644 index d5fc76ad3..000000000 --- a/src/runtime/interop37.cs +++ /dev/null @@ -1,149 +0,0 @@ -// Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. - - -#if PYTHON37 -using System; -using System.Collections; -using System.Collections.Specialized; -using System.Runtime.InteropServices; -using System.Reflection; -using System.Text; - -namespace Python.Runtime -{ - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } - - // Auto-generated from PyHeapTypeObject in Python.h - public static int ob_refcnt = 0; - public static int ob_type = 0; - public static int ob_size = 0; - public static int tp_name = 0; - public static int tp_basicsize = 0; - public static int tp_itemsize = 0; - public static int tp_dealloc = 0; - public static int tp_print = 0; - public static int tp_getattr = 0; - public static int tp_setattr = 0; - public static int tp_as_async = 0; - public static int tp_repr = 0; - public static int tp_as_number = 0; - public static int tp_as_sequence = 0; - public static int tp_as_mapping = 0; - public static int tp_hash = 0; - public static int tp_call = 0; - public static int tp_str = 0; - public static int tp_getattro = 0; - public static int tp_setattro = 0; - public static int tp_as_buffer = 0; - public static int tp_flags = 0; - public static int tp_doc = 0; - public static int tp_traverse = 0; - public static int tp_clear = 0; - public static int tp_richcompare = 0; - public static int tp_weaklistoffset = 0; - public static int tp_iter = 0; - public static int tp_iternext = 0; - public static int tp_methods = 0; - public static int tp_members = 0; - public static int tp_getset = 0; - public static int tp_base = 0; - public static int tp_dict = 0; - public static int tp_descr_get = 0; - public static int tp_descr_set = 0; - public static int tp_dictoffset = 0; - public static int tp_init = 0; - public static int tp_alloc = 0; - public static int tp_new = 0; - public static int tp_free = 0; - public static int tp_is_gc = 0; - public static int tp_bases = 0; - public static int tp_mro = 0; - public static int tp_cache = 0; - public static int tp_subclasses = 0; - public static int tp_weaklist = 0; - public static int tp_del = 0; - public static int tp_version_tag = 0; - public static int tp_finalize = 0; - public static int am_await = 0; - public static int am_aiter = 0; - public static int am_anext = 0; - public static int nb_add = 0; - public static int nb_subtract = 0; - public static int nb_multiply = 0; - public static int nb_remainder = 0; - public static int nb_divmod = 0; - public static int nb_power = 0; - public static int nb_negative = 0; - public static int nb_positive = 0; - public static int nb_absolute = 0; - public static int nb_bool = 0; - public static int nb_invert = 0; - public static int nb_lshift = 0; - public static int nb_rshift = 0; - public static int nb_and = 0; - public static int nb_xor = 0; - public static int nb_or = 0; - public static int nb_int = 0; - public static int nb_reserved = 0; - public static int nb_float = 0; - public static int nb_inplace_add = 0; - public static int nb_inplace_subtract = 0; - public static int nb_inplace_multiply = 0; - public static int nb_inplace_remainder = 0; - public static int nb_inplace_power = 0; - public static int nb_inplace_lshift = 0; - public static int nb_inplace_rshift = 0; - public static int nb_inplace_and = 0; - public static int nb_inplace_xor = 0; - public static int nb_inplace_or = 0; - public static int nb_floor_divide = 0; - public static int nb_true_divide = 0; - public static int nb_inplace_floor_divide = 0; - public static int nb_inplace_true_divide = 0; - public static int nb_index = 0; - public static int nb_matrix_multiply = 0; - public static int nb_inplace_matrix_multiply = 0; - public static int mp_length = 0; - public static int mp_subscript = 0; - public static int mp_ass_subscript = 0; - public static int sq_length = 0; - public static int sq_concat = 0; - public static int sq_repeat = 0; - public static int sq_item = 0; - public static int was_sq_slice = 0; - public static int sq_ass_item = 0; - public static int was_sq_ass_slice = 0; - public static int sq_contains = 0; - public static int sq_inplace_concat = 0; - public static int sq_inplace_repeat = 0; - public static int bf_getbuffer = 0; - public static int bf_releasebuffer = 0; - public static int name = 0; - public static int ht_slots = 0; - public static int qualname = 0; - public static int ht_cached_keys = 0; - - /* here are optional user slots, followed by the members. */ - public static int members = 0; - } -} - -#endif diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 8853c2d5e..3295ab110 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -29,7 +29,7 @@ public static IntPtr Initialize() /// public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { - var len = Runtime.PyTuple_Size(args); + int len = Runtime.PyTuple_Size(args); if (len < 3) { return Exceptions.RaiseTypeError("invalid argument list"); diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index b608b6c55..48dc1eb9a 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -291,7 +291,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth { // loop to find match, return invoker w/ or /wo error MethodBase[] _methods = null; - var pynargs = (int)Runtime.PyTuple_Size(args); + int pynargs = Runtime.PyTuple_Size(args); object arg; var isGeneric = false; ArrayList defaultArgList = null; @@ -313,9 +313,9 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth isGeneric = true; } ParameterInfo[] pi = mi.GetParameters(); - var clrnargs = pi.Length; + int clrnargs = pi.Length; var match = false; - var arrayStart = -1; + int arrayStart = -1; var outs = 0; if (pynargs == clrnargs) @@ -326,7 +326,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth { match = true; defaultArgList = new ArrayList(); - for (var v = pynargs; v < clrnargs; v++) + for (int v = pynargs; v < clrnargs; v++) { if (pi[v].DefaultValue == DBNull.Value) { @@ -350,7 +350,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth { var margs = new object[clrnargs]; - for (int n = 0; n < clrnargs; n++) + for (var n = 0; n < clrnargs; n++) { IntPtr op; if (n < pynargs) @@ -632,19 +632,6 @@ internal class MethodSorter : IComparer { int IComparer.Compare(object m1, object m2) { - var me1 = (MethodBase)m1; - var me2 = (MethodBase)m2; - if (me1.DeclaringType != me2.DeclaringType) - { - // m2's type derives from m1's type, favor m2 - if (me1.DeclaringType.IsAssignableFrom(me2.DeclaringType)) - return 1; - - // m1's type derives from m2's type, favor m1 - if (me2.DeclaringType.IsAssignableFrom(me1.DeclaringType)) - return -1; - } - int p1 = MethodBinder.GetPrecedence((MethodBase)m1); int p2 = MethodBinder.GetPrecedence((MethodBase)m2); if (p1 < p2) diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index f402f91f8..3985ed2cb 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -104,7 +104,7 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) { if (self.info.IsGenericMethod) { - var len = Runtime.PyTuple_Size(args); //FIXME: Never used + int len = Runtime.PyTuple_Size(args); //FIXME: Never used Type[] sigTp = Runtime.PythonArgsToTypeArray(args, true); if (sigTp != null) { @@ -129,7 +129,7 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) if (target == IntPtr.Zero && !self.m.IsStatic()) { - var len = Runtime.PyTuple_Size(args); + int len = Runtime.PyTuple_Size(args); if (len < 1) { Exceptions.SetError(Exceptions.TypeError, "not enough arguments"); diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 8af722d29..4accb1531 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -190,17 +190,10 @@ public void LoadNames() foreach (string name in AssemblyManager.GetNames(_namespace)) { cache.TryGetValue(name, out m); - if (m != null) + if (m == null) { - continue; + ManagedType attr = GetAttribute(name, true); } - IntPtr attr = Runtime.PyDict_GetItemString(dict, name); - // If __dict__ has already set a custom property, skip it. - if (attr != IntPtr.Zero) - { - continue; - } - GetAttribute(name, true); } } @@ -335,17 +328,6 @@ public CLRModule() : base("clr") } } - public static void Reset() - { - hacked = false; - interactive_preload = true; - preload = false; - - // XXX Test performance of new features // - _SuppressDocs = false; - _SuppressOverloads = false; - } - /// /// The initializing of the preload hook has to happen as late as /// possible since sys.ps1 is created after the CLR module is diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 3b8c71efa..0e075824a 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -519,9 +519,9 @@ public virtual void DelItem(int index) /// Returns the length for objects that support the Python sequence /// protocol, or 0 if the object does not support the protocol. /// - public virtual long Length() + public virtual int Length() { - var s = Runtime.PyObject_Size(obj); + int s = Runtime.PyObject_Size(obj); if (s < 0) { Runtime.PyErr_Clear(); diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 8e6957855..67f93c6e2 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -537,15 +537,10 @@ public void Dispose() public class PyScopeManager { - public static PyScopeManager Global; + public readonly static PyScopeManager Global = new PyScopeManager(); private Dictionary NamedScopes = new Dictionary(); - internal static void Reset() - { - Global = new PyScopeManager(); - } - internal PyScope NewScope(string name) { if (name == null) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index c1b663d22..a23c7ac79 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -140,9 +140,9 @@ public static void Initialize() Initialize(setSysArgv: true); } - public static void Initialize(bool setSysArgv = true, bool initSigs = false) + public static void Initialize(bool setSysArgv = true) { - Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs); + Initialize(Enumerable.Empty(), setSysArgv: setSysArgv); } /// @@ -153,9 +153,8 @@ public static void Initialize(bool setSysArgv = true, bool initSigs = false) /// more than once, though initialization will only happen on the /// first call. It is *not* necessary to hold the Python global /// interpreter lock (GIL) to call this method. - /// initSigs can be set to 1 to do default python signal configuration. This will override the way signals are handled by the application. /// - public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false) + public static void Initialize(IEnumerable args, bool setSysArgv = true) { if (!initialized) { @@ -165,20 +164,10 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, // during an initial "import clr", and the world ends shortly thereafter. // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). delegateManager = new DelegateManager(); - Runtime.Initialize(initSigs); + Runtime.Initialize(); initialized = true; Exceptions.Clear(); - // Make sure we clean up properly on app domain unload. - AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; - - // Remember to shut down the runtime. - AddShutdownHandler(Runtime.Shutdown); - - // The global scope gets used implicitly quite early on, remember - // to clear it out when we shut down. - AddShutdownHandler(PyScopeManager.Global.Clear); - if (setSysArgv) { Py.SetArgv(args); @@ -234,11 +223,6 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, } } - static void OnDomainUnload(object _, EventArgs __) - { - Shutdown(); - } - /// /// A helper to perform initialization from the context of an active /// CPython interpreter process - this bootstraps the managed runtime @@ -311,80 +295,18 @@ public static void Shutdown() if (initialized) { PyScopeManager.Global.Clear(); - - // If the shutdown handlers trigger a domain unload, - // don't call shutdown again. - AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; - - ExecuteShutdownHandlers(); + Marshal.FreeHGlobal(_pythonHome); + _pythonHome = IntPtr.Zero; + Marshal.FreeHGlobal(_programName); + _programName = IntPtr.Zero; + Marshal.FreeHGlobal(_pythonPath); + _pythonPath = IntPtr.Zero; + Runtime.Shutdown(); initialized = false; } } - /// - /// Called when the engine is shut down. - /// - /// Shutdown handlers are run in reverse order they were added, so that - /// resources available when running a shutdown handler are the same as - /// what was available when it was added. - /// - public delegate void ShutdownHandler(); - - static List ShutdownHandlers = new List(); - - /// - /// Add a function to be called when the engine is shut down. - /// - /// Shutdown handlers are executed in the opposite order they were - /// added, so that you can be sure that everything that was initialized - /// when you added the handler is still initialized when you need to shut - /// down. - /// - /// If the same shutdown handler is added several times, it will be run - /// several times. - /// - /// Don't add shutdown handlers while running a shutdown handler. - /// - public static void AddShutdownHandler(ShutdownHandler handler) - { - ShutdownHandlers.Add(handler); - } - - /// - /// Remove a shutdown handler. - /// - /// If the same shutdown handler is added several times, only the last - /// one is removed. - /// - /// Don't remove shutdown handlers while running a shutdown handler. - /// - public static void RemoveShutdownHandler(ShutdownHandler handler) - { - for (int index = ShutdownHandlers.Count - 1; index >= 0; --index) - { - if (ShutdownHandlers[index] == handler) - { - ShutdownHandlers.RemoveAt(index); - break; - } - } - } - - /// - /// Run all the shutdown handlers. - /// - /// They're run in opposite order they were added. - /// - static void ExecuteShutdownHandlers() - { - while(ShutdownHandlers.Count > 0) - { - var handler = ShutdownHandlers[ShutdownHandlers.Count - 1]; - ShutdownHandlers.RemoveAt(ShutdownHandlers.Count - 1); - handler(); - } - } /// /// AcquireLock Method diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 21a1d20fd..5ef06f3cc 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -2,7 +2,6 @@ using System.Runtime.InteropServices; using System.Security; using System.Text; -using System.Collections.Generic; namespace Python.Runtime { @@ -22,7 +21,7 @@ public static IntPtr LoadLibrary(string fileName) } #elif MONO_OSX private static int RTLD_GLOBAL = 0x8; - private const string NativeDll = "/usr/lib/libSystem.dylib"; + private const string NativeDll = "/usr/lib/libSystem.dylib" private static IntPtr RTLD_DEFAULT = new IntPtr(-2); public static IntPtr LoadLibrary(string fileName) @@ -149,7 +148,7 @@ public class Runtime #elif PYTHON36 internal const string _pyversion = "3.6"; internal const string _pyver = "36"; -#elif PYTHON37 +#elif PYTHON37 // TODO: Add `interop37.cs` after PY37 is released internal const string _pyversion = "3.7"; internal const string _pyver = "37"; #else @@ -196,68 +195,6 @@ public class Runtime // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows) internal static bool IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; - /// - /// Operating system type as reported by Python. - /// - public enum OperatingSystemType - { - Windows, - Darwin, - Linux, - Other - } - - static readonly Dictionary OperatingSystemTypeMapping = new Dictionary() - { - { "Windows", OperatingSystemType.Windows }, - { "Darwin", OperatingSystemType.Darwin }, - { "Linux", OperatingSystemType.Linux }, - }; - - /// - /// Gets the operating system as reported by python's platform.system(). - /// - public static OperatingSystemType OperatingSystem { get; private set; } - - /// - /// Gets the operating system as reported by python's platform.system(). - /// - public static string OperatingSystemName { get; private set; } - - public enum MachineType - { - i386, - x86_64, - Other - }; - - /// - /// Map lower-case version of the python machine name to the processor - /// type. There are aliases, e.g. x86_64 and amd64 are two names for - /// the same thing. Make sure to lower-case the search string, because - /// capitalization can differ. - /// - static readonly Dictionary MachineTypeMapping = new Dictionary() - { - ["i386"] = MachineType.i386, - ["i686"] = MachineType.i386, - ["x86"] = MachineType.i386, - ["x86_64"] = MachineType.x86_64, - ["amd64"] = MachineType.x86_64, - ["x64"] = MachineType.x86_64, - ["em64t"] = MachineType.x86_64, - }; - - /// - /// Gets the machine architecture as reported by python's platform.machine(). - /// - public static MachineType Machine { get; private set; }/* set in Initialize using python's platform.machine */ - - /// - /// Gets the machine architecture as reported by python's platform.machine(). - /// - public static string MachineName { get; private set; } - internal static bool IsPython2 = pyversionnumber < 30; internal static bool IsPython3 = pyversionnumber >= 30; @@ -269,11 +206,11 @@ public enum MachineType /// /// Initialize the runtime... /// - internal static void Initialize(bool initSigs = false) + internal static void Initialize() { if (Py_IsInitialized() == 0) { - Py_InitializeEx(initSigs ? 1 : 0); + Py_Initialize(); } if (PyEval_ThreadsInitialized() == 0) @@ -281,15 +218,6 @@ internal static void Initialize(bool initSigs = false) PyEval_InitThreads(); } - IsFinalizing = false; - - CLRModule.Reset(); - GenericUtil.Reset(); - PyScopeManager.Reset(); - ClassManager.Reset(); - ClassDerivedObject.Reset(); - TypeManager.Reset(); - IntPtr op; IntPtr dict; if (IsPython3) @@ -411,10 +339,6 @@ internal static void Initialize(bool initSigs = false) NativeMethods.FreeLibrary(dllLocal); } #endif - // Initialize data about the platform we're running on. We need - // this for the type manager and potentially other details. Must - // happen after caching the python types, above. - InitializePlatformData(); // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); @@ -432,53 +356,6 @@ internal static void Initialize(bool initSigs = false) AssemblyManager.UpdatePath(); } - /// - /// Initializes the data about platforms. - /// - /// This must be the last step when initializing the runtime: - /// GetManagedString needs to have the cached values for types. - /// But it must run before initializing anything outside the runtime - /// because those rely on the platform data. - /// - private static void InitializePlatformData() - { - IntPtr op; - IntPtr fn; - IntPtr platformModule = PyImport_ImportModule("platform"); - IntPtr emptyTuple = PyTuple_New(0); - - fn = PyObject_GetAttrString(platformModule, "system"); - op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - OperatingSystemName = GetManagedString(op); - XDecref(op); - XDecref(fn); - - fn = PyObject_GetAttrString(platformModule, "machine"); - op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - MachineName = GetManagedString(op); - XDecref(op); - XDecref(fn); - - XDecref(emptyTuple); - XDecref(platformModule); - - // Now convert the strings into enum values so we can do switch - // statements rather than constant parsing. - OperatingSystemType OSType; - if (!OperatingSystemTypeMapping.TryGetValue(OperatingSystemName, out OSType)) - { - OSType = OperatingSystemType.Other; - } - OperatingSystem = OSType; - - MachineType MType; - if (!MachineTypeMapping.TryGetValue(MachineName.ToLower(), out MType)) - { - MType = MachineType.Other; - } - Machine = MType; - } - internal static void Shutdown() { AssemblyManager.Shutdown(); @@ -549,7 +426,7 @@ internal static int AtExit() /// internal static void CheckExceptionOccurred() { - if (PyErr_Occurred() != IntPtr.Zero) + if (PyErr_Occurred() != 0) { throw new PythonException(); } @@ -557,7 +434,7 @@ internal static void CheckExceptionOccurred() internal static IntPtr ExtendTuple(IntPtr t, params IntPtr[] args) { - var size = PyTuple_Size(t); + int size = PyTuple_Size(t); int add = args.Length; IntPtr item; @@ -600,7 +477,7 @@ internal static Type[] PythonArgsToTypeArray(IntPtr arg, bool mangleObjects) free = true; } - var n = PyTuple_Size(args); + int n = PyTuple_Size(args); var types = new Type[n]; Type t = null; @@ -736,9 +613,6 @@ internal static unsafe long Refcount(IntPtr op) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void Py_Initialize(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_InitializeEx(int initsigs); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int Py_IsInitialized(); @@ -1034,13 +908,8 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_Not(IntPtr pointer); - internal static long PyObject_Size(IntPtr pointer) - { - return (long) _PyObject_Size(pointer); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyObject_Size")] - private static extern IntPtr _PyObject_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyObject_Size(IntPtr pointer); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_Hash(IntPtr op); @@ -1270,61 +1139,26 @@ internal static bool PyFloat_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern bool PySequence_Check(IntPtr pointer); - internal static IntPtr PySequence_GetItem(IntPtr pointer, long index) - { - return PySequence_GetItem(pointer, new IntPtr(index)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_GetItem(IntPtr pointer, IntPtr index); - - internal static int PySequence_SetItem(IntPtr pointer, long index, IntPtr value) - { - return PySequence_SetItem(pointer, new IntPtr(index), value); - } + internal static extern IntPtr PySequence_GetItem(IntPtr pointer, int index); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_SetItem(IntPtr pointer, IntPtr index, IntPtr value); - - internal static int PySequence_DelItem(IntPtr pointer, long index) - { - return PySequence_DelItem(pointer, new IntPtr(index)); - } + internal static extern int PySequence_SetItem(IntPtr pointer, int index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_DelItem(IntPtr pointer, IntPtr index); - - internal static IntPtr PySequence_GetSlice(IntPtr pointer, long i1, long i2) - { - return PySequence_GetSlice(pointer, new IntPtr(i1), new IntPtr(i2)); - } + internal static extern int PySequence_DelItem(IntPtr pointer, int index); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_GetSlice(IntPtr pointer, IntPtr i1, IntPtr i2); - - internal static int PySequence_SetSlice(IntPtr pointer, long i1, long i2, IntPtr v) - { - return PySequence_SetSlice(pointer, new IntPtr(i1), new IntPtr(i2), v); - } + internal static extern IntPtr PySequence_GetSlice(IntPtr pointer, int i1, int i2); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_SetSlice(IntPtr pointer, IntPtr i1, IntPtr i2, IntPtr v); - - internal static int PySequence_DelSlice(IntPtr pointer, long i1, long i2) - { - return PySequence_DelSlice(pointer, new IntPtr(i1), new IntPtr(i2)); - } + internal static extern int PySequence_SetSlice(IntPtr pointer, int i1, int i2, IntPtr v); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_DelSlice(IntPtr pointer, IntPtr i1, IntPtr i2); + internal static extern int PySequence_DelSlice(IntPtr pointer, int i1, int i2); - internal static long PySequence_Size(IntPtr pointer) - { - return (long) _PySequence_Size(pointer); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Size")] - private static extern IntPtr _PySequence_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PySequence_Size(IntPtr pointer); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PySequence_Contains(IntPtr pointer, IntPtr item); @@ -1332,24 +1166,14 @@ internal static long PySequence_Size(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PySequence_Concat(IntPtr pointer, IntPtr other); - internal static IntPtr PySequence_Repeat(IntPtr pointer, long count) - { - return PySequence_Repeat(pointer, new IntPtr(count)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_Repeat(IntPtr pointer, IntPtr count); + internal static extern IntPtr PySequence_Repeat(IntPtr pointer, int count); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PySequence_Index(IntPtr pointer, IntPtr item); - internal static long PySequence_Count(IntPtr pointer, IntPtr value) - { - return (long) _PySequence_Count(pointer, value); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Count")] - private static extern IntPtr _PySequence_Count(IntPtr pointer, IntPtr value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PySequence_Count(IntPtr pointer, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PySequence_Tuple(IntPtr pointer); @@ -1375,57 +1199,33 @@ internal static bool PyString_Check(IntPtr ob) internal static IntPtr PyString_FromString(string value) { -#if PYTHON3 - return PyUnicode_FromKindAndData(_UCS, value, value.Length); -#elif PYTHON2 return PyString_FromStringAndSize(value, value.Length); -#endif } #if PYTHON3 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyBytes_FromString(string op); - internal static long PyBytes_Size(IntPtr op) - { - return (long) _PyBytes_Size(op); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyBytes_Size")] - private static extern IntPtr _PyBytes_Size(IntPtr op); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyBytes_Size(IntPtr op); internal static IntPtr PyBytes_AS_STRING(IntPtr ob) { return ob + BytesOffset.ob_sval; } - internal static IntPtr PyString_FromStringAndSize(string value, long size) - { - return _PyString_FromStringAndSize(value, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyUnicode_FromStringAndSize")] - internal static extern IntPtr _PyString_FromStringAndSize( + internal static extern IntPtr PyString_FromStringAndSize( [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string value, - IntPtr size + int size ); - internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size) - { - return PyUnicode_FromStringAndSize(value, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyUnicode_FromStringAndSize(IntPtr value, IntPtr size); + internal static extern IntPtr PyUnicode_FromStringAndSize(IntPtr value, int size); #elif PYTHON2 - internal static IntPtr PyString_FromStringAndSize(string value, long size) - { - return PyString_FromStringAndSize(value, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyString_FromStringAndSize(string value, IntPtr size); + internal static extern IntPtr PyString_FromStringAndSize(string value, int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyString_AsString(IntPtr op); @@ -1446,30 +1246,20 @@ internal static bool PyUnicode_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err); - internal static IntPtr PyUnicode_FromKindAndData(int kind, string s, long size) - { - return PyUnicode_FromKindAndData(kind, s, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyUnicode_FromKindAndData( + internal static extern IntPtr PyUnicode_FromKindAndData( int kind, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UcsMarshaler))] string s, - IntPtr size + int size ); - internal static IntPtr PyUnicode_FromUnicode(string s, long size) + internal static IntPtr PyUnicode_FromUnicode(string s, int size) { return PyUnicode_FromKindAndData(_UCS, s, size); } - internal static long PyUnicode_GetSize(IntPtr ob) - { - return (long)_PyUnicode_GetSize(ob); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyUnicode_GetSize")] - private static extern IntPtr _PyUnicode_GetSize(IntPtr ob); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyUnicode_GetSize(IntPtr ob); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyUnicode_AsUnicode(IntPtr ob); @@ -1485,26 +1275,16 @@ internal static long PyUnicode_GetSize(IntPtr ob) EntryPoint = PyUnicodeEntryPoint + "FromEncodedObject")] internal static extern IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err); - internal static IntPtr PyUnicode_FromUnicode(string s, long size) - { - return PyUnicode_FromUnicode(s, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = PyUnicodeEntryPoint + "FromUnicode")] - private static extern IntPtr PyUnicode_FromUnicode( + internal static extern IntPtr PyUnicode_FromUnicode( [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UcsMarshaler))] string s, - IntPtr size + int size ); - internal static long PyUnicode_GetSize(IntPtr ob) - { - return (long) _PyUnicode_GetSize(ob); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = PyUnicodeEntryPoint + "GetSize")] - internal static extern IntPtr _PyUnicode_GetSize(IntPtr ob); + internal static extern int PyUnicode_GetSize(IntPtr ob); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = PyUnicodeEntryPoint + "AsUnicode")] @@ -1547,7 +1327,7 @@ internal static string GetManagedString(IntPtr op) if (type == PyUnicodeType) { IntPtr p = PyUnicode_AsUnicode(op); - int length = (int)PyUnicode_GetSize(op); + int length = PyUnicode_GetSize(op); int size = length * _UCS; var buffer = new byte[size]; @@ -1613,13 +1393,8 @@ internal static bool PyDict_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyDict_Clear(IntPtr pointer); - internal static long PyDict_Size(IntPtr pointer) - { - return (long) _PyDict_Size(pointer); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyDict_Size")] - internal static extern IntPtr _PyDict_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyDict_Size(IntPtr pointer); //==================================================================== @@ -1631,40 +1406,20 @@ internal static bool PyList_Check(IntPtr ob) return PyObject_TYPE(ob) == PyListType; } - internal static IntPtr PyList_New(long size) - { - return PyList_New(new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_New(IntPtr size); + internal static extern IntPtr PyList_New(int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyList_AsTuple(IntPtr pointer); - internal static IntPtr PyList_GetItem(IntPtr pointer, long index) - { - return PyList_GetItem(pointer, new IntPtr(index)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_GetItem(IntPtr pointer, IntPtr index); - - internal static int PyList_SetItem(IntPtr pointer, long index, IntPtr value) - { - return PyList_SetItem(pointer, new IntPtr(index), value); - } + internal static extern IntPtr PyList_GetItem(IntPtr pointer, int index); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_SetItem(IntPtr pointer, IntPtr index, IntPtr value); - - internal static int PyList_Insert(IntPtr pointer, long index, IntPtr value) - { - return PyList_Insert(pointer, new IntPtr(index), value); - } + internal static extern int PyList_SetItem(IntPtr pointer, int index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_Insert(IntPtr pointer, IntPtr index, IntPtr value); + internal static extern int PyList_Insert(IntPtr pointer, int index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyList_Append(IntPtr pointer, IntPtr value); @@ -1675,29 +1430,15 @@ internal static int PyList_Insert(IntPtr pointer, long index, IntPtr value) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyList_Sort(IntPtr pointer); - internal static IntPtr PyList_GetSlice(IntPtr pointer, long start, long end) - { - return PyList_GetSlice(pointer, new IntPtr(start), new IntPtr(end)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_GetSlice(IntPtr pointer, IntPtr start, IntPtr end); - - internal static int PyList_SetSlice(IntPtr pointer, long start, long end, IntPtr value) - { - return PyList_SetSlice(pointer, new IntPtr(start), new IntPtr(end), value); - } + internal static extern IntPtr PyList_GetSlice(IntPtr pointer, int start, int end); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_SetSlice(IntPtr pointer, IntPtr start, IntPtr end, IntPtr value); + internal static extern int PyList_SetSlice(IntPtr pointer, int start, int end, IntPtr value); - internal static long PyList_Size(IntPtr pointer) - { - return (long) _PyList_Size(pointer); - } + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyList_Size(IntPtr pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyList_Size")] - private static extern IntPtr _PyList_Size(IntPtr pointer); //==================================================================== // Python tuple API @@ -1708,45 +1449,20 @@ internal static bool PyTuple_Check(IntPtr ob) return PyObject_TYPE(ob) == PyTupleType; } - internal static IntPtr PyTuple_New(long size) - { - return PyTuple_New(new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_New(IntPtr size); - - internal static IntPtr PyTuple_GetItem(IntPtr pointer, long index) - { - return PyTuple_GetItem(pointer, new IntPtr(index)); - } + internal static extern IntPtr PyTuple_New(int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_GetItem(IntPtr pointer, IntPtr index); - - internal static int PyTuple_SetItem(IntPtr pointer, long index, IntPtr value) - { - return PyTuple_SetItem(pointer, new IntPtr(index), value); - } + internal static extern IntPtr PyTuple_GetItem(IntPtr pointer, int index); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyTuple_SetItem(IntPtr pointer, IntPtr index, IntPtr value); - - internal static IntPtr PyTuple_GetSlice(IntPtr pointer, long start, long end) - { - return PyTuple_GetSlice(pointer, new IntPtr(start), new IntPtr(end)); - } + internal static extern int PyTuple_SetItem(IntPtr pointer, int index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_GetSlice(IntPtr pointer, IntPtr start, IntPtr end); + internal static extern IntPtr PyTuple_GetSlice(IntPtr pointer, int start, int end); - internal static long PyTuple_Size(IntPtr pointer) - { - return (long) _PyTuple_Size(pointer); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyTuple_Size")] - private static extern IntPtr _PyTuple_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyTuple_Size(IntPtr pointer); //==================================================================== @@ -1852,13 +1568,8 @@ internal static bool PyObject_TypeCheck(IntPtr ob, IntPtr tp) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kw); - internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) - { - return PyType_GenericAlloc(type, new IntPtr(n)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n); + internal static extern IntPtr PyType_GenericAlloc(IntPtr type, int n); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyType_Ready(IntPtr type); @@ -1892,21 +1603,11 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) // Python memory API //==================================================================== - internal static IntPtr PyMem_Malloc(long size) - { - return PyMem_Malloc(new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyMem_Malloc(IntPtr size); - - internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) - { - return PyMem_Realloc(ptr, new IntPtr(size)); - } + internal static extern IntPtr PyMem_Malloc(int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyMem_Realloc(IntPtr ptr, IntPtr size); + internal static extern IntPtr PyMem_Realloc(IntPtr ptr, int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyMem_Free(IntPtr ptr); @@ -1938,7 +1639,7 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) internal static extern void PyErr_NormalizeException(IntPtr ob, IntPtr val, IntPtr tb); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyErr_Occurred(); + internal static extern int PyErr_Occurred(); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_Fetch(ref IntPtr ob, ref IntPtr val, ref IntPtr tb); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index d19c8737f..6570ee083 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Reflection; @@ -21,10 +21,6 @@ static TypeManager() cache = new Dictionary(128); } - public static void Reset() - { - cache = new Dictionary(128); - } /// /// Given a managed Type derived from ExtensionType, get the handle to @@ -451,225 +447,6 @@ internal static IntPtr AllocateTypeObject(string name) } - #region Native Code Page - /// - /// Initialized by InitializeNativeCodePage. - /// - /// This points to a page of memory allocated using mmap or VirtualAlloc - /// (depending on the system), and marked read and execute (not write). - /// Very much on purpose, the page is *not* released on a shutdown and - /// is instead leaked. See the TestDomainReload test case. - /// - /// The contents of the page are two native functions: one that returns 0, - /// one that returns 1. - /// - /// If python didn't keep its gc list through a Py_Finalize we could remove - /// this entire section. - /// - internal static IntPtr NativeCodePage = IntPtr.Zero; - - /// - /// Structure to describe native code. - /// - /// Use NativeCode.Active to get the native code for the current platform. - /// - /// Generate the code by creating the following C code: - /// - /// int Return0() { return 0; } - /// int Return1() { return 1; } - /// - /// Then compiling on the target platform, e.g. with gcc or clang: - /// cc -c -fomit-frame-pointer -O2 foo.c - /// And then analyzing the resulting functions with a hex editor, e.g.: - /// objdump -disassemble foo.o - /// - internal class NativeCode - { - /// - /// The code, as a string of bytes. - /// - public byte[] Code { get; private set; } - - /// - /// Where does the "return 0" function start? - /// - public int Return0 { get; private set; } - - /// - /// Where does the "return 1" function start? - /// - public int Return1 { get; private set; } - - public static NativeCode Active - { - get - { - switch(Runtime.Machine) - { - case Runtime.MachineType.i386: - return I386; - case Runtime.MachineType.x86_64: - return X86_64; - default: - throw new NotImplementedException($"No support for {Runtime.MachineName}"); - } - } - } - - /// - /// Code for x86_64. See the class comment for how it was generated. - /// - public static readonly NativeCode X86_64 = new NativeCode() - { - Return0 = 0x10, - Return1 = 0, - Code = new byte[] - { - // First Return1: - 0xb8, 0x01, 0x00, 0x00, 0x00, // movl $1, %eax - 0xc3, // ret - - // Now some padding so that Return0 can be 16-byte-aligned. - // I put Return1 first so there's not as much padding to type in. - 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, // nop - - // Now Return0. - 0x31, 0xc0, // xorl %eax, %eax - 0xc3, // ret - } - }; - - /// - /// Code for X86. - /// - /// It's bitwise identical to X86_64, so we just point to it. - /// - /// - public static readonly NativeCode I386 = X86_64; - } - - /// - /// Platform-dependent mmap and mprotect. - /// - internal interface IMemoryMapper - { - /// - /// Map at least numBytes of memory. Mark the page read-write (but not exec). - /// - IntPtr MapWriteable(int numBytes); - - /// - /// Sets the mapped memory to be read-exec (but not write). - /// - void SetReadExec(IntPtr mappedMemory, int numBytes); - } - - class WindowsMemoryMapper : IMemoryMapper - { - const UInt32 MEM_COMMIT = 0x1000; - const UInt32 MEM_RESERVE = 0x2000; - const UInt32 PAGE_READWRITE = 0x04; - const UInt32 PAGE_EXECUTE_READ = 0x20; - - [DllImport("kernel32.dll")] - static extern IntPtr VirtualAlloc(IntPtr lpAddress, IntPtr dwSize, UInt32 flAllocationType, UInt32 flProtect); - - [DllImport("kernel32.dll")] - static extern bool VirtualProtect(IntPtr lpAddress, IntPtr dwSize, UInt32 flNewProtect, out UInt32 lpflOldProtect); - - public IntPtr MapWriteable(int numBytes) - { - return VirtualAlloc(IntPtr.Zero, new IntPtr(numBytes), - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - UInt32 _; - VirtualProtect(mappedMemory, new IntPtr(numBytes), PAGE_EXECUTE_READ, out _); - } - } - - class UnixMemoryMapper : IMemoryMapper - { - const int PROT_READ = 0x1; - const int PROT_WRITE = 0x2; - const int PROT_EXEC = 0x4; - - const int MAP_PRIVATE = 0x2; - int MAP_ANONYMOUS - { - get - { - switch (Runtime.OperatingSystem) - { - case Runtime.OperatingSystemType.Darwin: - return 0x1000; - case Runtime.OperatingSystemType.Linux: - return 0x20; - default: - throw new NotImplementedException($"mmap is not supported on {Runtime.OperatingSystemName}"); - } - } - } - - [DllImport("libc")] - static extern IntPtr mmap(IntPtr addr, IntPtr len, int prot, int flags, int fd, IntPtr offset); - - [DllImport("libc")] - static extern int mprotect(IntPtr addr, IntPtr len, int prot); - - public IntPtr MapWriteable(int numBytes) - { - // MAP_PRIVATE must be set on linux, even though MAP_ANON implies it. - // It doesn't hurt on darwin, so just do it. - return mmap(IntPtr.Zero, new IntPtr(numBytes), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, IntPtr.Zero); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - mprotect(mappedMemory, new IntPtr(numBytes), PROT_READ | PROT_EXEC); - } - } - - internal static IMemoryMapper CreateMemoryMapper() - { - switch (Runtime.OperatingSystem) - { - case Runtime.OperatingSystemType.Darwin: - case Runtime.OperatingSystemType.Linux: - return new UnixMemoryMapper(); - case Runtime.OperatingSystemType.Windows: - return new WindowsMemoryMapper(); - default: - throw new NotImplementedException($"No support for {Runtime.OperatingSystemName}"); - } - } - - /// - /// Initializes the native code page. - /// - /// Safe to call if we already initialized (this function is idempotent). - /// - /// - internal static void InitializeNativeCodePage() - { - // Do nothing if we already initialized. - if (NativeCodePage != IntPtr.Zero) - { - return; - } - - // Allocate the page, write the native code into it, then set it - // to be executable. - IMemoryMapper mapper = CreateMemoryMapper(); - int codeLength = NativeCode.Active.Code.Length; - NativeCodePage = mapper.MapWriteable(codeLength); - Marshal.Copy(NativeCode.Active.Code, 0, NativeCodePage, codeLength); - mapper.SetReadExec(NativeCodePage, codeLength); - } -#endregion - /// /// Given a newly allocated Python type object and a managed Type that /// provides the implementation for the type, connect the type slots of @@ -677,10 +454,8 @@ internal static void InitializeNativeCodePage() /// internal static void InitializeSlots(IntPtr type, Type impl) { - // We work from the most-derived class up; make sure to get - // the most-derived slot and not to override it with a base - // class's slot. - var seen = new HashSet(); + var seen = new Hashtable(8); + Type offsetType = typeof(TypeOffset); while (impl != null) { @@ -698,52 +473,24 @@ internal static void InitializeSlots(IntPtr type, Type impl) continue; } - if (seen.Contains(name)) + if (seen[name] != null) { continue; } - InitializeSlot(type, Interop.GetThunk(method), name); + FieldInfo fi = offsetType.GetField(name); + var offset = (int)fi.GetValue(offsetType); + + IntPtr slot = Interop.GetThunk(method); + Marshal.WriteIntPtr(type, offset, slot); - seen.Add(name); + seen[name] = 1; } impl = impl.BaseType; } - - // See the TestDomainReload test: there was a crash related to - // the gc-related slots. They always return 0 or 1 because we don't - // really support gc: - // tp_traverse (returns 0) - // tp_clear (returns 0) - // tp_is_gc (returns 1) - // We can't do without: python really wants those slots to exist. - // We can't implement those in C# because the application domain - // can be shut down and the memory released. - InitializeNativeCodePage(); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return0, "tp_traverse"); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return0, "tp_clear"); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return1, "tp_is_gc"); } - /// - /// Helper for InitializeSlots. - /// - /// Initializes one slot to point to a function pointer. - /// The function pointer might be a thunk for C#, or it may be - /// an address in the NativeCodePage. - /// - /// Type being initialized. - /// Function pointer. - /// Name of the method. - static void InitializeSlot(IntPtr type, IntPtr slot, string name) - { - Type typeOffset = typeof(TypeOffset); - FieldInfo fi = typeOffset.GetField(name); - var offset = (int)fi.GetValue(typeOffset); - - Marshal.WriteIntPtr(type, offset, slot); - } /// /// Given a newly allocated Python type object and a managed Type that diff --git a/src/testing/InheritanceTest.cs b/src/testing/InheritanceTest.cs deleted file mode 100644 index 529703b3c..000000000 --- a/src/testing/InheritanceTest.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; - -namespace Python.Test -{ - public class BaseClass - { - public bool IsBase() => true; - } - - public class DerivedClass : BaseClass - { - public new bool IsBase() => false; - } -} diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index 27639ed5a..8a8d9ed2b 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -83,7 +83,6 @@ - @@ -111,4 +110,4 @@ - \ No newline at end of file + diff --git a/src/testing/conversiontest.cs b/src/testing/conversiontest.cs index 06ab7cb4e..36294594c 100644 --- a/src/testing/conversiontest.cs +++ b/src/testing/conversiontest.cs @@ -60,19 +60,4 @@ public string GetValue() return value; } } - - public class UnicodeString - { - public string value = "안녕"; - - public string GetString() - { - return value; - } - - public override string ToString() - { - return value; - } - } } diff --git a/src/tests/importtest.py b/src/tests/importtest.py deleted file mode 100644 index fe93764e9..000000000 --- a/src/tests/importtest.py +++ /dev/null @@ -1,13 +0,0 @@ -# -*- coding: utf-8 -*- - -import sys -try: - del sys.modules["System.IO"] -except KeyError: - pass - -assert "FileStream" not in globals() -import System.IO -from System.IO import * - -assert "FileStream" in globals() diff --git a/src/tests/test_class.py b/src/tests/test_class.py index 612ce442e..68773508b 100644 --- a/src/tests/test_class.py +++ b/src/tests/test_class.py @@ -281,14 +281,3 @@ def PyCallback(self, self2): testobj.DoCallback() assert testobj.PyCallbackWasCalled assert testobj.SameReference - - -def test_method_inheritance(): - """Ensure that we call the overridden method instead of the one provided in - the base class.""" - - base = Test.BaseClass() - derived = Test.DerivedClass() - - assert base.IsBase() == True - assert derived.IsBase() == False diff --git a/src/tests/test_conversion.py b/src/tests/test_conversion.py index 0ba10a80e..53e5d8051 100644 --- a/src/tests/test_conversion.py +++ b/src/tests/test_conversion.py @@ -2,12 +2,11 @@ """Test CLR <-> Python type conversions.""" -from __future__ import unicode_literals import System import pytest -from Python.Test import ConversionTest, UnicodeString +from Python.Test import ConversionTest -from ._compat import indexbytes, long, unichr, text_type, PY2, PY3 +from ._compat import indexbytes, long, unichr def test_bool_conversion(): @@ -536,14 +535,6 @@ def test_string_conversion(): with pytest.raises(TypeError): ConversionTest().StringField = 1 - - world = UnicodeString() - test_unicode_str = u"안녕" - assert test_unicode_str == text_type(world.value) - assert test_unicode_str == text_type(world.GetString()) - # TODO: not sure what to do for Python 2 here (GH PR #670) - if PY3: - assert test_unicode_str == text_type(world) def test_interface_conversion(): diff --git a/src/tests/test_exceptions.py b/src/tests/test_exceptions.py index 08b00d77d..c697290ee 100644 --- a/src/tests/test_exceptions.py +++ b/src/tests/test_exceptions.py @@ -289,8 +289,7 @@ def test_python_compat_of_managed_exceptions(): assert e.args == (msg,) assert isinstance(e.args, tuple) if PY3: - strexp = "OverflowException('Simple message" - assert repr(e)[:len(strexp)] == strexp + assert repr(e) == "OverflowException('Simple message',)" elif PY2: assert repr(e) == "OverflowException(u'Simple message',)" diff --git a/src/tests/test_import.py b/src/tests/test_import.py index 25877be15..42cafc4df 100644 --- a/src/tests/test_import.py +++ b/src/tests/test_import.py @@ -3,7 +3,7 @@ """Test the import statement.""" import pytest -import sys + def test_relative_missing_import(): """Test that a relative missing import doesn't crash. @@ -11,12 +11,3 @@ def test_relative_missing_import(): Relative import in the site-packages folder""" with pytest.raises(ImportError): from . import _missing_import - - -def test_import_all_on_second_time(): - """Test import all attributes after a normal import without '*'. - Due to import * only allowed at module level, the test body splited - to a module file.""" - from . import importtest - del sys.modules[importtest.__name__] - diff --git a/src/tests/test_subclass.py b/src/tests/test_subclass.py index ab440d429..43d013c7c 100644 --- a/src/tests/test_subclass.py +++ b/src/tests/test_subclass.py @@ -128,39 +128,6 @@ def test_derived_class(): assert id(x) == id(ob) -def test_derived_traceback(): - """Test python exception traceback in class derived from managed base""" - class DerivedClass(SubClassTest): - __namespace__ = "Python.Test.traceback" - - def foo(self): - print (xyzname) - return None - - import sys,traceback - ob = DerivedClass() - - # direct call - try: - ob.foo() - assert False - except: - e = sys.exc_info() - assert "xyzname" in str(e[1]) - location = traceback.extract_tb(e[2])[-1] - assert location[2] == "foo" - - # call through managed code - try: - FunctionsTest.test_foo(ob) - assert False - except: - e = sys.exc_info() - assert "xyzname" in str(e[1]) - location = traceback.extract_tb(e[2])[-1] - assert location[2] == "foo" - - def test_create_instance(): """Test derived instances can be created from managed code""" DerivedClass = derived_class_fixture(test_create_instance.__name__) diff --git a/tools/geninterop/fake_libc_include/crypt.h b/tools/geninterop/fake_libc_include/crypt.h deleted file mode 100644 index 3b16d481f..000000000 --- a/tools/geninterop/fake_libc_include/crypt.h +++ /dev/null @@ -1 +0,0 @@ -#include "features.h" \ No newline at end of file diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py index f8ef8e561..bf5fdb96b 100644 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -173,8 +173,7 @@ def preprocess_python_headers(): "-D", "__attribute__(x)=", "-D", "__inline__=inline", "-D", "__asm__=;#pragma asm", - "-D", "__int64=long long", - "-D", "_POSIX_THREADS" + "-D", "__int64=long long" ] if hasattr(sys, "abiflags"): @@ -186,7 +185,7 @@ def preprocess_python_headers(): defines.extend(("-D", "PYTHON_WITH_WIDE_UNICODE")) python_h = os.path.join(include_py, "Python.h") - cmd = ["clang", "-pthread", "-I"] + include_dirs + defines + ["-E", python_h] + cmd = ["clang", "-I"] + include_dirs + defines + ["-E", python_h] # normalize as the parser doesn't like windows line endings. lines = [] From f4a047b9108d1d759372d8a9e590e8940b10bf8a Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Fri, 18 Jan 2019 18:46:41 -0300 Subject: [PATCH 24/76] Performance improvements ClrObject - Dispose PyObj - Increasing minor version to 14 - Adding `UnsafeDispose()` for `PyObject` which does not require acquiring/releasing the lock - Adding check before calling `SetArgsAndCause` for the `ClrObject`, the call only makes sense when we are an exception and causes an overhead --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/clrobject.cs | 12 ++++++++---- src/runtime/pyobject.cs | 19 ++++++++++++++++++- src/runtime/resources/clr.py | 2 +- 8 files changed, 32 insertions(+), 11 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 2518f77fb..31cc3ea8d 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.13 +current_version = 1.0.5.14 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 3aa2d1a62..e1b225169 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.13" + version: "1.0.5.14" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 3695e33fa..ddb59747c 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.13", + version="1.0.5.14", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index e9d39ba33..1df774b4a 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.13")] +[assembly: AssemblyVersion("1.0.5.14")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index c5b789a18..3384b6c5e 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.12"), + Version = new Version("1.0.5.14"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index fb3d0e0d7..17782f026 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Runtime.InteropServices; namespace Python.Runtime @@ -29,9 +29,13 @@ internal CLRObject(object ob, IntPtr tp) gcHandle = gc; inst = ob; - // Fix the BaseException args (and __cause__ in case of Python 3) - // slot if wrapping a CLR exception - Exceptions.SetArgsAndCause(py); + // for performance before calling SetArgsAndCause() lets check if we are an exception + if (inst is Exception) + { + // Fix the BaseException args (and __cause__ in case of Python 3) + // slot if wrapping a CLR exception + Exceptions.SetArgsAndCause(py); + } } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 0e075824a..35a4e6fa9 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -101,7 +101,7 @@ public object AsManagedObject(Type t) } return result; } - + /// /// As Method /// @@ -156,6 +156,23 @@ public void Dispose() GC.SuppressFinalize(this); } + /// + /// Unsafe Dispose Method. + /// To be used when already owning the GIL lock. + /// + public void UnsafeDispose() + { + if (!disposed) + { + if (!Runtime.IsFinalizing) + { + Runtime.XDecref(obj); + obj = IntPtr.Zero; + } + disposed = true; + } + GC.SuppressFinalize(this); + } /// /// GetPythonType Method diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 7caff08f4..48d2907c4 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.13" +__version__ = "1.0.5.14" class clrproperty(object): From daddc6809573123f930380bb29e8d40922d05329 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Tue, 22 Jan 2019 21:56:03 +0000 Subject: [PATCH 25/76] Version bump to 1.0.5.15 Version bump to match nuget package --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 31cc3ea8d..ccb9c32e8 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.14 +current_version = 1.0.5.15 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index e1b225169..0b70bedc9 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.14" + version: "1.0.5.15" build: skip: True # [not win] diff --git a/setup.py b/setup.py index ddb59747c..421054d77 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.14", + version="1.0.5.15", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index 1df774b4a..cc71683aa 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.14")] +[assembly: AssemblyVersion("1.0.5.15")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 3384b6c5e..143759f2e 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.14"), + Version = new Version("1.0.5.15"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 48d2907c4..f877058e8 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.14" +__version__ = "1.0.5.15" class clrproperty(object): From f47b9f5c4900c01fe7ccc253af8ef65de2e813e6 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Wed, 20 Feb 2019 13:02:42 -0300 Subject: [PATCH 26/76] Improve Indicator Performance Benchmark - Adding new `interop` `type` cache holding a `bool`, true if its an `exception`. - Adding a `setter` and `getter` cache for the `propertyobject`. --- src/runtime/interop.cs | 26 ++++++++--- src/runtime/propertyobject.cs | 88 +++++++++++++++++++++++++++++++++-- src/runtime/runtime.cs | 11 +++-- 3 files changed, 112 insertions(+), 13 deletions(-) diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 4ae4b61e0..7ead0b1d2 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -1,6 +1,6 @@ using System; using System.Collections; -using System.Collections.Specialized; +using System.Collections.Generic; using System.Runtime.InteropServices; using System.Reflection; using System.Text; @@ -88,8 +88,7 @@ static ObjectOffset() public static int magic(IntPtr ob) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) + if (IsException(ob)) { return ExceptionOffset.ob_data; } @@ -98,8 +97,7 @@ public static int magic(IntPtr ob) public static int DictOffset(IntPtr ob) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) + if (IsException(ob)) { return ExceptionOffset.ob_dict; } @@ -108,8 +106,7 @@ public static int DictOffset(IntPtr ob) public static int Size(IntPtr ob) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) + if (IsException(ob)) { return ExceptionOffset.Size(); } @@ -128,6 +125,21 @@ public static int Size(IntPtr ob) public static int ob_type; private static int ob_dict; private static int ob_data; + private static readonly Dictionary ExceptionTypeCache = new Dictionary(); + + private static bool IsException(IntPtr pyObject) + { + bool res; + var type = Runtime.PyObject_TYPE(pyObject); + if (!ExceptionTypeCache.TryGetValue(type, out res)) + { + res = Runtime.PyObjectType_TypeCheck(type, Exceptions.BaseException) + || Runtime.PyObjectType_TypeCheck(type, Runtime.PyTypeType) + && Runtime.PyType_IsSubtype(pyObject, Exceptions.BaseException); + ExceptionTypeCache.Add(type, res); + } + return res; + } } [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] diff --git a/src/runtime/propertyobject.cs b/src/runtime/propertyobject.cs index f2c97f163..00a8ab4e7 100644 --- a/src/runtime/propertyobject.cs +++ b/src/runtime/propertyobject.cs @@ -1,4 +1,5 @@ using System; +using System.Linq.Expressions; using System.Reflection; using System.Security.Permissions; @@ -12,6 +13,10 @@ internal class PropertyObject : ExtensionType private PropertyInfo info; private MethodInfo getter; private MethodInfo setter; + private bool getterCacheFailed; + private bool setterCacheFailed; + private Func getterCache; + private Action setterCache; [StrongNameIdentityPermission(SecurityAction.Assert)] public PropertyObject(PropertyInfo md) @@ -67,7 +72,21 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { - result = self.info.GetValue(co.inst, null); + if (self.getterCache == null && !self.getterCacheFailed) + { + // if the getter is not public 'GetGetMethod' will not find it + // but calling 'GetValue' will work, so for backwards compatibility + // we will use it instead + self.getterCache = BuildGetter(self.info); + if (self.getterCache == null) + { + self.getterCacheFailed = true; + } + } + + result = self.getterCacheFailed ? + self.info.GetValue(co.inst, null) + : self.getterCache(co.inst); return Converter.ToPython(result, self.info.PropertyType); } catch (Exception e) @@ -81,7 +100,6 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) } } - /// /// Descriptor __set__ implementation. This method sets the value of /// a property based on the given Python value. The Python value must @@ -132,7 +150,27 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) Exceptions.RaiseTypeError("invalid target"); return -1; } - self.info.SetValue(co.inst, newval, null); + + if (self.setterCache == null && !self.setterCacheFailed) + { + // if the setter is not public 'GetSetMethod' will not find it + // but calling 'SetValue' will work, so for backwards compatibility + // we will use it instead + self.setterCache = BuildSetter(self.info); + if (self.setterCache == null) + { + self.setterCacheFailed = true; + } + } + + if (self.setterCacheFailed) + { + self.info.SetValue(co.inst, newval, null); + } + else + { + self.setterCache(co.inst, newval); + } } else { @@ -160,5 +198,49 @@ public static IntPtr tp_repr(IntPtr ob) var self = (PropertyObject)GetManagedObject(ob); return Runtime.PyString_FromString($""); } + + private static Func BuildGetter(PropertyInfo propertyInfo) + { + var methodInfo = propertyInfo.GetGetMethod(); + if (methodInfo == null) + { + // if the getter is not public 'GetGetMethod' will not find it + return null; + } + var obj = Expression.Parameter(typeof(object), "o"); + // Require because we will know at runtime the declaring type + // so 'obj' is declared as typeof(object) + var instance = Expression.Convert(obj, methodInfo.DeclaringType); + + var expressionCall = Expression.Call(instance, methodInfo); + + return Expression.Lambda>( + Expression.Convert(expressionCall, typeof(object)), + obj).Compile(); + } + + private static Action BuildSetter(PropertyInfo propertyInfo) + { + var methodInfo = propertyInfo.GetSetMethod(); + if (methodInfo == null) + { + // if the setter is not public 'GetSetMethod' will not find it + return null; + } + var obj = Expression.Parameter(typeof(object), "o"); + // Require because we will know at runtime the declaring type + // so 'obj' is declared as typeof(object) + var instance = Expression.Convert(obj, methodInfo.DeclaringType); + + var value = Expression.Parameter(typeof(object)); + var argument = Expression.Convert(value, methodInfo.GetParameters()[0].ParameterType); + + var expressionCall = Expression.Call(instance, methodInfo, argument); + + return Expression.Lambda>( + expressionCall, + obj, + value).Compile(); + } } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 5ef06f3cc..d1183f68e 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -104,7 +104,7 @@ public static IntPtr GetProcAddress(IntPtr dllHandle, string name) public class Runtime { // C# compiler copies constants to the assemblies that references this library. - // We needs to replace all public constants to static readonly fields to allow + // We needs to replace all public constants to static readonly fields to allow // binary substitution of different Python.Runtime.dll builds in a target application. public static int UCS => _UCS; @@ -130,7 +130,7 @@ public class Runtime #endif // C# compiler copies constants to the assemblies that references this library. - // We needs to replace all public constants to static readonly fields to allow + // We needs to replace all public constants to static readonly fields to allow // binary substitution of different Python.Runtime.dll builds in a target application. public static string pyversion => _pyversion; @@ -173,7 +173,7 @@ public class Runtime #endif // C# compiler copies constants to the assemblies that references this library. - // We needs to replace all public constants to static readonly fields to allow + // We needs to replace all public constants to static readonly fields to allow // binary substitution of different Python.Runtime.dll builds in a target application. public static readonly string PythonDLL = _PythonDll; @@ -1565,6 +1565,11 @@ internal static bool PyObject_TypeCheck(IntPtr ob, IntPtr tp) return (t == tp) || PyType_IsSubtype(t, tp); } + internal static bool PyObjectType_TypeCheck(IntPtr type, IntPtr tp) + { + return (type == tp) || PyType_IsSubtype(type, tp); + } + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kw); From ab0d1b35c05f9e067a3ff60f607c18faf21f8577 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Wed, 20 Feb 2019 15:50:37 -0300 Subject: [PATCH 27/76] Review: Adding parameter count check --- src/runtime/propertyobject.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/runtime/propertyobject.cs b/src/runtime/propertyobject.cs index 00a8ab4e7..cc6125157 100644 --- a/src/runtime/propertyobject.cs +++ b/src/runtime/propertyobject.cs @@ -232,8 +232,13 @@ private static Action BuildSetter(PropertyInfo propertyInfo) // so 'obj' is declared as typeof(object) var instance = Expression.Convert(obj, methodInfo.DeclaringType); + var parameters = methodInfo.GetParameters(); + if (parameters.Length != 1) + { + return null; + } var value = Expression.Parameter(typeof(object)); - var argument = Expression.Convert(value, methodInfo.GetParameters()[0].ParameterType); + var argument = Expression.Convert(value, parameters[0].ParameterType); var expressionCall = Expression.Call(instance, methodInfo, argument); From 5c35e800f7423ab9b474978ccea87f387530039d Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Wed, 20 Feb 2019 16:47:52 +0000 Subject: [PATCH 28/76] Decimal Parsing Allows Numeric String in Exponential Notation --- src/runtime/converter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index d8ec3fcd2..a81d9fb99 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -941,7 +941,7 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo op = Runtime.PyObject_Str(value); decimal m; string sm = Runtime.GetManagedString(op); - if (!Decimal.TryParse(sm, NumberStyles.Number, nfi, out m)) + if (!Decimal.TryParse(sm, NumberStyles.Number | NumberStyles.AllowExponent, nfi, out m)) { goto type_error; } From 7e9c3df3a8365e744aa581c3c2700669190e054a Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Wed, 20 Feb 2019 16:54:18 +0000 Subject: [PATCH 29/76] Version bump to 1.0.5.16 Version bump to match nuget package --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ccb9c32e8..7ea53f6ba 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.15 +current_version = 1.0.5.16 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 0b70bedc9..497adf5a0 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.15" + version: "1.0.5.16" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 421054d77..03fe25456 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.15", + version="1.0.5.16", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index cc71683aa..c17676f61 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.15")] +[assembly: AssemblyVersion("1.0.5.16")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 143759f2e..b198df56b 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.15"), + Version = new Version("1.0.5.16"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index f877058e8..c31118458 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.15" +__version__ = "1.0.5.16" class clrproperty(object): From b88e72f099d2331bc768b74f81bfcc9fab158ab5 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Thu, 21 Feb 2019 15:14:22 -0300 Subject: [PATCH 30/76] Remove ExceptionTypeCache - Removing `ExceptionTypeCache` which was causing issues and fix for issue caused performance overhead. --- src/runtime/interop.cs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 7ead0b1d2..5168bcd98 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -125,20 +125,13 @@ public static int Size(IntPtr ob) public static int ob_type; private static int ob_dict; private static int ob_data; - private static readonly Dictionary ExceptionTypeCache = new Dictionary(); private static bool IsException(IntPtr pyObject) { - bool res; var type = Runtime.PyObject_TYPE(pyObject); - if (!ExceptionTypeCache.TryGetValue(type, out res)) - { - res = Runtime.PyObjectType_TypeCheck(type, Exceptions.BaseException) - || Runtime.PyObjectType_TypeCheck(type, Runtime.PyTypeType) - && Runtime.PyType_IsSubtype(pyObject, Exceptions.BaseException); - ExceptionTypeCache.Add(type, res); - } - return res; + return Runtime.PyObjectType_TypeCheck(type, Exceptions.BaseException) + || Runtime.PyObjectType_TypeCheck(type, Runtime.PyTypeType) + && Runtime.PyType_IsSubtype(pyObject, Exceptions.BaseException); } } From 3ce460841c5d9422636cf00571b63ee94fad46c4 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Thu, 21 Feb 2019 15:23:52 -0300 Subject: [PATCH 31/76] Bumping version to 1.5.0.17 --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 7ea53f6ba..4be598ac8 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.16 +current_version = 1.0.5.17 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 497adf5a0..1e85fffac 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.16" + version: "1.0.5.17" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 03fe25456..42230e099 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.16", + version="1.0.5.17", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index c17676f61..ba4cfde67 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.16")] +[assembly: AssemblyVersion("1.0.5.17")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index b198df56b..2d08d31ee 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.16"), + Version = new Version("1.0.5.17"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index c31118458..279d7f4f2 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.16" +__version__ = "1.0.5.17" class clrproperty(object): From 26e88d748c80c9bc585d3acc1077fe5e0530ea6f Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 24 Jun 2018 03:20:44 +0800 Subject: [PATCH 32/76] Finalizer for PyObject --- src/embed_tests/TestFinalizer.cs | 143 +++++++++++++++++++++++++++++++ src/runtime/delegatemanager.cs | 29 ++++--- src/runtime/finalizer.cs | 117 +++++++++++++++++++++++++ src/runtime/pyobject.cs | 14 +-- src/runtime/pyscope.cs | 12 +-- src/runtime/pythonexception.cs | 24 +++--- src/runtime/runtime.cs | 11 +++ 7 files changed, 319 insertions(+), 31 deletions(-) create mode 100644 src/embed_tests/TestFinalizer.cs create mode 100644 src/runtime/finalizer.cs diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs new file mode 100644 index 000000000..03e2f1be5 --- /dev/null +++ b/src/embed_tests/TestFinalizer.cs @@ -0,0 +1,143 @@ +using NUnit.Framework; +using Python.Runtime; +using System; +using System.Threading; + +namespace Python.EmbeddingTest +{ + public class TestFinalizer + { + private string _PYTHONMALLOC = string.Empty; + + [SetUp] + public void SetUp() + { + try + { + _PYTHONMALLOC = Environment.GetEnvironmentVariable("PYTHONMALLOC"); + } + catch (ArgumentNullException) + { + _PYTHONMALLOC = string.Empty; + } + Environment.SetEnvironmentVariable("PYTHONMALLOC", "malloc"); + PythonEngine.Initialize(); + } + + [TearDown] + public void TearDown() + { + PythonEngine.Shutdown(); + if (string.IsNullOrEmpty(_PYTHONMALLOC)) + { + Environment.SetEnvironmentVariable("PYTHONMALLOC", _PYTHONMALLOC); + } + } + + private static void FullGCCollect() + { + GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced); + GC.WaitForFullGCComplete(); + GC.WaitForPendingFinalizers(); + } + + [Test] + public void CollectBasicObject() + { + int thId = Thread.CurrentThread.ManagedThreadId; + Finalizer.Instance.Threshold = 1; + bool called = false; + EventHandler handler = (s, e) => + { + Assert.AreEqual(thId, Thread.CurrentThread.ManagedThreadId); + Assert.GreaterOrEqual(e.ObjectCount, 1); + called = true; + }; + Finalizer.Instance.CollectOnce += handler; + FullGCCollect(); + PyLong obj = new PyLong(1024); + + WeakReference shortWeak = new WeakReference(obj); + WeakReference longWeak = new WeakReference(obj, true); + obj = null; + FullGCCollect(); + // The object has been resurrected + Assert.IsFalse(shortWeak.IsAlive); + Assert.IsTrue(longWeak.IsAlive); + + Assert.IsFalse(called); + var garbage = Finalizer.Instance.GetCollectedObjects(); + // FIXME: If make some query for garbage, + // the above case will failed Assert.IsFalse(shortWeak.IsAlive) + //Assert.IsTrue(garbage.All(T => T.IsAlive)); + + Finalizer.Instance.CallPendingFinalizers(); + Assert.IsTrue(called); + + FullGCCollect(); + //Assert.IsFalse(garbage.All(T => T.IsAlive)); + + Assert.IsNull(longWeak.Target); + + Finalizer.Instance.CollectOnce -= handler; + } + + private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) + { + // Must larger than 512 bytes make sure Python use + string str = new string('1', 1024); + Finalizer.Instance.Enable = true; + FullGCCollect(); + FullGCCollect(); + pyCollect.Invoke(); + Finalizer.Instance.CallPendingFinalizers(); + Finalizer.Instance.Enable = enbale; + + // Estimate unmanaged memory size + long before = Environment.WorkingSet - GC.GetTotalMemory(true); + for (int i = 0; i < 10000; i++) + { + // Memory will leak when disable Finalizer + new PyString(str); + } + FullGCCollect(); + FullGCCollect(); + pyCollect.Invoke(); + if (enbale) + { + Finalizer.Instance.CallPendingFinalizers(); + } + + FullGCCollect(); + FullGCCollect(); + long after = Environment.WorkingSet - GC.GetTotalMemory(true); + return after - before; + + } + + /// + /// Because of two vms both have their memory manager, + /// this test only prove the finalizer has take effect. + /// + [Test] + [Ignore("Too many uncertainties, only manual on when debugging")] + public void SimpleTestMemory() + { + bool oldState = Finalizer.Instance.Enable; + try + { + using (PyObject gcModule = PythonEngine.ImportModule("gc")) + using (PyObject pyCollect = gcModule.GetAttr("collect")) + { + long span1 = CompareWithFinalizerOn(pyCollect, false); + long span2 = CompareWithFinalizerOn(pyCollect, true); + Assert.Less(span2, span1); + } + } + finally + { + Finalizer.Instance.Enable = oldState; + } + } + } +} diff --git a/src/runtime/delegatemanager.cs b/src/runtime/delegatemanager.cs index 7632816d1..706bd4bc4 100644 --- a/src/runtime/delegatemanager.cs +++ b/src/runtime/delegatemanager.cs @@ -181,10 +181,12 @@ A possible alternate strategy would be to create custom subclasses too "special" for this to work. It would be more work, so for now the 80/20 rule applies :) */ - public class Dispatcher + public class Dispatcher : IDisposable { public IntPtr target; public Type dtype; + private bool _disposed = false; + private bool _finalized = false; public Dispatcher(IntPtr target, Type dtype) { @@ -195,18 +197,25 @@ public Dispatcher(IntPtr target, Type dtype) ~Dispatcher() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; + if (_finalized || _disposed) + { + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); + } - // Note: the managed GC thread can run and try to free one of - // these *after* the Python runtime has been finalized! - if (Runtime.Py_IsInitialized() > 0) + public void Dispose() + { + if (_disposed) { - IntPtr gs = PythonEngine.AcquireLock(); - Runtime.XDecref(target); - PythonEngine.ReleaseLock(gs); + return; } + _disposed = true; + Runtime.XDecref(target); + target = IntPtr.Zero; + dtype = null; + GC.SuppressFinalize(this); } public object Dispatch(ArrayList args) diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs new file mode 100644 index 000000000..344260f97 --- /dev/null +++ b/src/runtime/finalizer.cs @@ -0,0 +1,117 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; +using System.Threading; + +namespace Python.Runtime +{ + public class Finalizer + { + public class CollectArgs : EventArgs + { + public int ObjectCount { get; set; } + } + + public static readonly Finalizer Instance = new Finalizer(); + + public event EventHandler CollectOnce; + + private ConcurrentQueue _objQueue = new ConcurrentQueue(); + + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + private delegate int PedingCall(IntPtr arg); + private PedingCall _collectAction; + private bool _pending = false; + private readonly object _collectingLock = new object(); + public int Threshold { get; set; } + public bool Enable { get; set; } + + private Finalizer() + { + Enable = true; + Threshold = 200; + _collectAction = OnCollect; + } + + public void CallPendingFinalizers() + { + if (Thread.CurrentThread.ManagedThreadId != Runtime.MainManagedThreadId) + { + throw new Exception("PendingCall should execute in main Python thread"); + } + Runtime.Py_MakePendingCalls(); + } + + public List GetCollectedObjects() + { + return _objQueue.Select(T => new WeakReference(T)).ToList(); + } + + internal void AddFinalizedObject(IDisposable obj) + { + if (!Enable) + { + return; + } + if (Runtime.Py_IsInitialized() == 0) + { + // XXX: Memory will leak if a PyObject finalized after Python shutdown, + // for avoiding that case, user should call GC.Collect manual before shutdown. + return; + } + _objQueue.Enqueue(obj); + GC.ReRegisterForFinalize(obj); + if (_objQueue.Count >= Threshold) + { + Collect(); + } + } + + internal static void Shutdown() + { + Instance.DisposeAll(); + Instance.CallPendingFinalizers(); + Runtime.PyErr_Clear(); + } + + private void Collect() + { + lock (_collectingLock) + { + if (_pending) + { + return; + } + _pending = true; + } + IntPtr func = Marshal.GetFunctionPointerForDelegate(_collectAction); + if (Runtime.Py_AddPendingCall(func, IntPtr.Zero) != 0) + { + // Full queue, append next time + _pending = false; + } + } + + private int OnCollect(IntPtr arg) + { + CollectOnce?.Invoke(this, new CollectArgs() + { + ObjectCount = _objQueue.Count + }); + DisposeAll(); + _pending = false; + return 0; + } + + private void DisposeAll() + { + IDisposable obj; + while (_objQueue.TryDequeue(out obj)) + { + obj.Dispose(); + } + } + } +} diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 35a4e6fa9..22f6d3e88 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -17,6 +17,7 @@ public class PyObject : DynamicObject, IEnumerable, IDisposable { protected internal IntPtr obj = IntPtr.Zero; private bool disposed = false; + private bool _finalized = false; /// /// PyObject Constructor @@ -41,14 +42,15 @@ protected PyObject() // Ensure that encapsulated Python object is decref'ed appropriately // when the managed wrapper is garbage-collected. - ~PyObject() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; - - Dispose(); + if (_finalized || disposed) + { + return; + } + // Prevent a infinity loop by calling GC.WaitForPendingFinalizers + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); } diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 67f93c6e2..32d9626bd 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -37,6 +37,7 @@ public class PyScope : DynamicObject, IDisposable internal readonly IntPtr variables; private bool _isDisposed; + private bool _finalized = false; /// /// The Manager this scope associated with. @@ -527,11 +528,12 @@ public void Dispose() ~PyScope() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; - - Dispose(); + if (_finalized || _isDisposed) + { + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); } } diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index ded7fbeb5..1b166854a 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -6,7 +6,7 @@ namespace Python.Runtime /// Provides a managed interface to exceptions thrown by the Python /// runtime. /// - public class PythonException : System.Exception + public class PythonException : System.Exception, IDisposable { private IntPtr _pyType = IntPtr.Zero; private IntPtr _pyValue = IntPtr.Zero; @@ -15,6 +15,7 @@ public class PythonException : System.Exception private string _message = ""; private string _pythonTypeName = ""; private bool disposed = false; + private bool _finalized = false; public PythonException() { @@ -45,11 +46,13 @@ public PythonException() } if (_pyTB != IntPtr.Zero) { - PyObject tb_module = PythonEngine.ImportModule("traceback"); - Runtime.XIncref(_pyTB); - using (var pyTB = new PyObject(_pyTB)) + using (PyObject tb_module = PythonEngine.ImportModule("traceback")) { - _tb = tb_module.InvokeMethod("format_tb", pyTB).ToString(); + Runtime.XIncref(_pyTB); + using (var pyTB = new PyObject(_pyTB)) + { + _tb = tb_module.InvokeMethod("format_tb", pyTB).ToString(); + } } } PythonEngine.ReleaseLock(gs); @@ -60,11 +63,12 @@ public PythonException() ~PythonException() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; - - Dispose(); + if (_finalized || disposed) + { + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); } /// diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index d1183f68e..85346790d 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -2,6 +2,7 @@ using System.Runtime.InteropServices; using System.Security; using System.Text; +using System.Threading; namespace Python.Runtime { @@ -198,6 +199,8 @@ public class Runtime internal static bool IsPython2 = pyversionnumber < 30; internal static bool IsPython3 = pyversionnumber >= 30; + public static int MainManagedThreadId { get; internal set; } + /// /// Encoding to use to convert Unicode to/from Managed to Native /// @@ -211,6 +214,7 @@ internal static void Initialize() if (Py_IsInitialized() == 0) { Py_Initialize(); + MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; } if (PyEval_ThreadsInitialized() == 0) @@ -358,6 +362,7 @@ internal static void Initialize() internal static void Shutdown() { + Finalizer.Shutdown(); AssemblyManager.Shutdown(); Exceptions.Shutdown(); ImportHook.Shutdown(); @@ -1668,5 +1673,11 @@ internal static bool PyObjectType_TypeCheck(IntPtr type, IntPtr tp) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyMethod_Function(IntPtr ob); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int Py_AddPendingCall(IntPtr func, IntPtr arg); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int Py_MakePendingCalls(); } } From fccd1d874f6ade3490cb8597ee42d0dc526e7f86 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 24 Jun 2018 15:40:14 +0800 Subject: [PATCH 33/76] Avoid test interdependency --- src/embed_tests/TestPyScope.cs | 101 ++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 46 deletions(-) diff --git a/src/embed_tests/TestPyScope.cs b/src/embed_tests/TestPyScope.cs index 49c15a3a1..21c0d2b3f 100644 --- a/src/embed_tests/TestPyScope.cs +++ b/src/embed_tests/TestPyScope.cs @@ -293,24 +293,27 @@ public void TestImportScopeByName() [Test] public void TestVariables() { - (ps.Variables() as dynamic)["ee"] = new PyInt(200); - var a0 = ps.Get("ee"); - Assert.AreEqual(200, a0); + using (Py.GIL()) + { + (ps.Variables() as dynamic)["ee"] = new PyInt(200); + var a0 = ps.Get("ee"); + Assert.AreEqual(200, a0); - ps.Exec("locals()['ee'] = 210"); - var a1 = ps.Get("ee"); - Assert.AreEqual(210, a1); + ps.Exec("locals()['ee'] = 210"); + var a1 = ps.Get("ee"); + Assert.AreEqual(210, a1); - ps.Exec("globals()['ee'] = 220"); - var a2 = ps.Get("ee"); - Assert.AreEqual(220, a2); + ps.Exec("globals()['ee'] = 220"); + var a2 = ps.Get("ee"); + Assert.AreEqual(220, a2); - using (var item = ps.Variables()) - { - item["ee"] = new PyInt(230); + using (var item = ps.Variables()) + { + item["ee"] = new PyInt(230); + } + var a3 = ps.Get("ee"); + Assert.AreEqual(230, a3); } - var a3 = ps.Get("ee"); - Assert.AreEqual(230, a3); } /// @@ -324,49 +327,55 @@ public void TestThread() //should be removed. dynamic _ps = ps; var ts = PythonEngine.BeginAllowThreads(); - using (Py.GIL()) - { - _ps.res = 0; - _ps.bb = 100; - _ps.th_cnt = 0; - //add function to the scope - //can be call many times, more efficient than ast - ps.Exec( - "def update():\n" + - " global res, th_cnt\n" + - " res += bb + 1\n" + - " th_cnt += 1\n" - ); - } - int th_cnt = 3; - for (int i =0; i< th_cnt; i++) + try { - System.Threading.Thread th = new System.Threading.Thread(()=> + using (Py.GIL()) + { + _ps.res = 0; + _ps.bb = 100; + _ps.th_cnt = 0; + //add function to the scope + //can be call many times, more efficient than ast + ps.Exec( + "def update():\n" + + " global res, th_cnt\n" + + " res += bb + 1\n" + + " th_cnt += 1\n" + ); + } + int th_cnt = 3; + for (int i = 0; i < th_cnt; i++) + { + System.Threading.Thread th = new System.Threading.Thread(() => + { + using (Py.GIL()) + { + //ps.GetVariable("update")(); //call the scope function dynamicly + _ps.update(); + } + }); + th.Start(); + } + //equivalent to Thread.Join, make the main thread join the GIL competition + int cnt = 0; + while (cnt != th_cnt) { using (Py.GIL()) { - //ps.GetVariable("update")(); //call the scope function dynamicly - _ps.update(); + cnt = ps.Get("th_cnt"); } - }); - th.Start(); - } - //equivalent to Thread.Join, make the main thread join the GIL competition - int cnt = 0; - while(cnt != th_cnt) - { + System.Threading.Thread.Sleep(10); + } using (Py.GIL()) { - cnt = ps.Get("th_cnt"); + var result = ps.Get("res"); + Assert.AreEqual(101 * th_cnt, result); } - System.Threading.Thread.Sleep(10); } - using (Py.GIL()) + finally { - var result = ps.Get("res"); - Assert.AreEqual(101* th_cnt, result); + PythonEngine.EndAllowThreads(ts); } - PythonEngine.EndAllowThreads(ts); } } } From 928115731e77c2cd7e5b1d484be40ae336772e55 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 24 Jun 2018 15:51:39 +0800 Subject: [PATCH 34/76] Add source to .csproj --- src/embed_tests/Python.EmbeddingTest.csproj | 249 +++++++-------- src/embed_tests/TestFinalizer.cs | 6 +- src/runtime/Python.Runtime.csproj | 327 ++++++++++---------- 3 files changed, 294 insertions(+), 288 deletions(-) diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index 66e8c7165..11ef2daac 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -1,125 +1,126 @@ - - - - Debug - AnyCPU - {4165C59D-2822-499F-A6DB-EACA4C331EB5} - Library - Python.EmbeddingTest - Python.EmbeddingTest - bin\Python.EmbeddingTest.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - 6 - true - prompt - - - x86 - - - x64 - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - - - ..\..\packages\NUnit.3.7.1\lib\net40\nunit.framework.dll - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Python.Runtime - - - - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - + + + + Debug + AnyCPU + {4165C59D-2822-499F-A6DB-EACA4C331EB5} + Library + Python.EmbeddingTest + Python.EmbeddingTest + bin\Python.EmbeddingTest.xml + bin\ + v4.0 + + 1591 + ..\..\ + $(SolutionDir)\bin\ + 6 + true + prompt + + + x86 + + + x64 + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + + + ..\..\packages\NUnit.3.7.1\lib\net40\nunit.framework.dll + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {097B4AC0-74E9-4C58-BCF8-C69746EC8271} + Python.Runtime + + + + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + \ No newline at end of file diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 03e2f1be5..6041e76cc 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -44,6 +44,8 @@ private static void FullGCCollect() [Test] public void CollectBasicObject() { + Assert.IsTrue(Finalizer.Instance.Enable); + int thId = Thread.CurrentThread.ManagedThreadId; Finalizer.Instance.Threshold = 1; bool called = false; @@ -62,11 +64,13 @@ public void CollectBasicObject() obj = null; FullGCCollect(); // The object has been resurrected - Assert.IsFalse(shortWeak.IsAlive); + // FIXME: Sometimes the shortWeak would get alive + //Assert.IsFalse(shortWeak.IsAlive); Assert.IsTrue(longWeak.IsAlive); Assert.IsFalse(called); var garbage = Finalizer.Instance.GetCollectedObjects(); + Assert.NotZero(garbage.Count); // FIXME: If make some query for garbage, // the above case will failed Assert.IsFalse(shortWeak.IsAlive) //Assert.IsTrue(garbage.All(T => T.IsAlive)); diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 1fea78082..28ea6424f 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,169 +1,170 @@ - - - - Debug - AnyCPU - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Library - Python.Runtime - Python.Runtime - bin\Python.Runtime.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - Properties - 6 - true - false - ..\pythonnet.snk - - - + + + + Debug + AnyCPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271} + Library + Python.Runtime + Python.Runtime + bin\Python.Runtime.xml + bin\ + v4.0 + + 1591 + ..\..\ + $(SolutionDir)\bin\ + Properties + 6 + true + false + ..\pythonnet.snk + + + - - PYTHON2;PYTHON27;UCS4 - true - pdbonly - - - PYTHON3;PYTHON36;UCS4 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS4;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON36;UCS4;TRACE;DEBUG - false - full - - - PYTHON2;PYTHON27;UCS2 - true - pdbonly - - - PYTHON3;PYTHON36;UCS2 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS2;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON36;UCS2;TRACE;DEBUG - false - full - - - - - - - - Properties\SharedAssemblyInfo.cs - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - clr.py - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - + --> + + PYTHON2;PYTHON27;UCS4 + true + pdbonly + + + PYTHON3;PYTHON36;UCS4 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS4;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON36;UCS4;TRACE;DEBUG + false + full + + + PYTHON2;PYTHON27;UCS2 + true + pdbonly + + + PYTHON3;PYTHON36;UCS2 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS2;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON36;UCS2;TRACE;DEBUG + false + full + + + + + + + + + Properties\SharedAssemblyInfo.cs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + clr.py + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + \ No newline at end of file From 481cb4f15bca461bca86fa84b96569471636d815 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 24 Jun 2018 16:11:05 +0800 Subject: [PATCH 35/76] Make sure recover the environment --- src/embed_tests/TestFinalizer.cs | 55 ++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 6041e76cc..7cf3b7309 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -22,6 +22,7 @@ public void SetUp() } Environment.SetEnvironmentVariable("PYTHONMALLOC", "malloc"); PythonEngine.Initialize(); + Exceptions.Clear(); } [TearDown] @@ -56,34 +57,40 @@ public void CollectBasicObject() called = true; }; Finalizer.Instance.CollectOnce += handler; - FullGCCollect(); - PyLong obj = new PyLong(1024); - - WeakReference shortWeak = new WeakReference(obj); - WeakReference longWeak = new WeakReference(obj, true); - obj = null; - FullGCCollect(); - // The object has been resurrected - // FIXME: Sometimes the shortWeak would get alive - //Assert.IsFalse(shortWeak.IsAlive); - Assert.IsTrue(longWeak.IsAlive); - - Assert.IsFalse(called); - var garbage = Finalizer.Instance.GetCollectedObjects(); - Assert.NotZero(garbage.Count); - // FIXME: If make some query for garbage, - // the above case will failed Assert.IsFalse(shortWeak.IsAlive) - //Assert.IsTrue(garbage.All(T => T.IsAlive)); + try + { + FullGCCollect(); + PyLong obj = new PyLong(1024); + + WeakReference shortWeak = new WeakReference(obj); + WeakReference longWeak = new WeakReference(obj, true); + obj = null; + FullGCCollect(); + // The object has been resurrected + // FIXME: Sometimes the shortWeak would get alive + //Assert.IsFalse(shortWeak.IsAlive); + Assert.IsTrue(longWeak.IsAlive); + + Assert.IsFalse(called); + var garbage = Finalizer.Instance.GetCollectedObjects(); + Assert.NotZero(garbage.Count); + // FIXME: If make some query for garbage, + // the above case will failed Assert.IsFalse(shortWeak.IsAlive) + //Assert.IsTrue(garbage.All(T => T.IsAlive)); - Finalizer.Instance.CallPendingFinalizers(); - Assert.IsTrue(called); + Finalizer.Instance.CallPendingFinalizers(); + Assert.IsTrue(called); - FullGCCollect(); - //Assert.IsFalse(garbage.All(T => T.IsAlive)); + FullGCCollect(); + //Assert.IsFalse(garbage.All(T => T.IsAlive)); - Assert.IsNull(longWeak.Target); + Assert.IsNull(longWeak.Target); + } + finally + { + Finalizer.Instance.CollectOnce -= handler; + } - Finalizer.Instance.CollectOnce -= handler; } private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) From 1959a6a87c091e7aa9eacc31733e149134ffb8b2 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 24 Jun 2018 16:52:34 +0800 Subject: [PATCH 36/76] Add StackTrace of C# exception --- src/runtime/pythonexception.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 1b166854a..4b4d8b107 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -136,7 +136,7 @@ public override string Message /// public override string StackTrace { - get { return _tb; } + get { return _tb + base.StackTrace; } } /// From e7b8635731eab8ba0fa928536b46abc2f808d9a1 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 24 Jun 2018 20:42:14 +0800 Subject: [PATCH 37/76] Clean up the test and interface --- src/embed_tests/TestFinalizer.cs | 55 +++++++++++++++++--------------- src/runtime/finalizer.cs | 29 +++++++++++------ 2 files changed, 49 insertions(+), 35 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 7cf3b7309..d9e9729fb 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -1,6 +1,7 @@ using NUnit.Framework; using Python.Runtime; using System; +using System.Linq; using System.Threading; namespace Python.EmbeddingTest @@ -8,6 +9,7 @@ namespace Python.EmbeddingTest public class TestFinalizer { private string _PYTHONMALLOC = string.Empty; + private int _oldThreshold; [SetUp] public void SetUp() @@ -21,6 +23,7 @@ public void SetUp() _PYTHONMALLOC = string.Empty; } Environment.SetEnvironmentVariable("PYTHONMALLOC", "malloc"); + _oldThreshold = Finalizer.Instance.Threshold; PythonEngine.Initialize(); Exceptions.Clear(); } @@ -28,6 +31,7 @@ public void SetUp() [TearDown] public void TearDown() { + Finalizer.Instance.Threshold = _oldThreshold; PythonEngine.Shutdown(); if (string.IsNullOrEmpty(_PYTHONMALLOC)) { @@ -56,41 +60,42 @@ public void CollectBasicObject() Assert.GreaterOrEqual(e.ObjectCount, 1); called = true; }; - Finalizer.Instance.CollectOnce += handler; - try - { - FullGCCollect(); - PyLong obj = new PyLong(1024); - WeakReference shortWeak = new WeakReference(obj); - WeakReference longWeak = new WeakReference(obj, true); - obj = null; - FullGCCollect(); - // The object has been resurrected - // FIXME: Sometimes the shortWeak would get alive - //Assert.IsFalse(shortWeak.IsAlive); - Assert.IsTrue(longWeak.IsAlive); + WeakReference shortWeak; + WeakReference longWeak; + { + MakeAGarbage(out shortWeak, out longWeak); + } + FullGCCollect(); + // The object has been resurrected + Assert.IsFalse(shortWeak.IsAlive); + Assert.IsTrue(longWeak.IsAlive); - Assert.IsFalse(called); + { var garbage = Finalizer.Instance.GetCollectedObjects(); Assert.NotZero(garbage.Count); - // FIXME: If make some query for garbage, - // the above case will failed Assert.IsFalse(shortWeak.IsAlive) - //Assert.IsTrue(garbage.All(T => T.IsAlive)); + Assert.IsTrue(garbage.Any(T => ReferenceEquals(T.Target, longWeak.Target))); + } + Assert.IsFalse(called); + Finalizer.Instance.CollectOnce += handler; + try + { Finalizer.Instance.CallPendingFinalizers(); - Assert.IsTrue(called); - - FullGCCollect(); - //Assert.IsFalse(garbage.All(T => T.IsAlive)); - - Assert.IsNull(longWeak.Target); } finally { Finalizer.Instance.CollectOnce -= handler; } + Assert.IsTrue(called); + } + private static void MakeAGarbage(out WeakReference shortWeak, out WeakReference longWeak) + { + PyLong obj = new PyLong(1024); + shortWeak = new WeakReference(obj); + longWeak = new WeakReference(obj, true); + obj = null; } private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) @@ -101,7 +106,7 @@ private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) FullGCCollect(); FullGCCollect(); pyCollect.Invoke(); - Finalizer.Instance.CallPendingFinalizers(); + Finalizer.Instance.Collect(); Finalizer.Instance.Enable = enbale; // Estimate unmanaged memory size @@ -116,7 +121,7 @@ private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) pyCollect.Invoke(); if (enbale) { - Finalizer.Instance.CallPendingFinalizers(); + Finalizer.Instance.Collect(); } FullGCCollect(); diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index 344260f97..64faaf5b4 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -22,7 +22,8 @@ public class CollectArgs : EventArgs [UnmanagedFunctionPointer(CallingConvention.Cdecl)] private delegate int PedingCall(IntPtr arg); - private PedingCall _collectAction; + private readonly PedingCall _collectAction; + private bool _pending = false; private readonly object _collectingLock = new object(); public int Threshold { get; set; } @@ -32,7 +33,7 @@ private Finalizer() { Enable = true; Threshold = 200; - _collectAction = OnCollect; + _collectAction = OnPendingCollect; } public void CallPendingFinalizers() @@ -44,6 +45,14 @@ public void CallPendingFinalizers() Runtime.Py_MakePendingCalls(); } + public void Collect() + { + using (var gilState = new Py.GILState()) + { + DisposeAll(); + } + } + public List GetCollectedObjects() { return _objQueue.Select(T => new WeakReference(T)).ToList(); @@ -65,7 +74,7 @@ internal void AddFinalizedObject(IDisposable obj) GC.ReRegisterForFinalize(obj); if (_objQueue.Count >= Threshold) { - Collect(); + AddPendingCollect(); } } @@ -76,7 +85,7 @@ internal static void Shutdown() Runtime.PyErr_Clear(); } - private void Collect() + private void AddPendingCollect() { lock (_collectingLock) { @@ -94,19 +103,19 @@ private void Collect() } } - private int OnCollect(IntPtr arg) + private int OnPendingCollect(IntPtr arg) { - CollectOnce?.Invoke(this, new CollectArgs() - { - ObjectCount = _objQueue.Count - }); - DisposeAll(); + Collect(); _pending = false; return 0; } private void DisposeAll() { + CollectOnce?.Invoke(this, new CollectArgs() + { + ObjectCount = _objQueue.Count + }); IDisposable obj; while (_objQueue.TryDequeue(out obj)) { From c51cebdac184e9403f198aa271907a343b1ff3dd Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 25 Jun 2018 00:09:31 +0800 Subject: [PATCH 38/76] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dc668b0c..f9db05d81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,8 +19,10 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Catches exceptions thrown in C# iterators (yield returns) and rethrows them in python ([#475][i475])([#693][p693]) - Implemented GetDynamicMemberNames() for PyObject to allow dynamic object members to be visible in the debugger ([#443][i443])([#690][p690]) - Incorporated reference-style links to issues and pull requests in the CHANGELOG ([#608][i608]) +- Added PyObject finalizer support, Python objects referred by C# can be auto collect now. ### Changed +- PythonException included C# call stack ### Fixed From 833d36777c4dc4cc27f9be851755bbb154e1c58e Mon Sep 17 00:00:00 2001 From: civilx64 <26513472+civilx64@users.noreply.github.com> Date: Sun, 1 Jul 2018 14:02:53 -0400 Subject: [PATCH 39/76] Add links to issues and PRs in CHANGELOG From 9aa9b1a9d352141b5bf2f7b3a904558b2af855ff Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 25 Jun 2018 00:22:23 +0800 Subject: [PATCH 40/76] Mono doesn't have GC.WaitForFullGCComplete --- src/embed_tests/TestFinalizer.cs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index d9e9729fb..985ce3b35 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -20,7 +20,7 @@ public void SetUp() } catch (ArgumentNullException) { - _PYTHONMALLOC = string.Empty; + _PYTHONMALLOC = null; } Environment.SetEnvironmentVariable("PYTHONMALLOC", "malloc"); _oldThreshold = Finalizer.Instance.Threshold; @@ -33,16 +33,12 @@ public void TearDown() { Finalizer.Instance.Threshold = _oldThreshold; PythonEngine.Shutdown(); - if (string.IsNullOrEmpty(_PYTHONMALLOC)) - { - Environment.SetEnvironmentVariable("PYTHONMALLOC", _PYTHONMALLOC); - } + Environment.SetEnvironmentVariable("PYTHONMALLOC", _PYTHONMALLOC); } private static void FullGCCollect() { GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced); - GC.WaitForFullGCComplete(); GC.WaitForPendingFinalizers(); } From 338dcc184ae3876623445fc833b79e8fe0bd2f98 Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 25 Jun 2018 16:52:16 +0800 Subject: [PATCH 41/76] Fixed PythonException leak --- src/runtime/finalizer.cs | 2 +- src/runtime/pythonexception.cs | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index 64faaf5b4..2f4549256 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -105,7 +105,7 @@ private void AddPendingCollect() private int OnPendingCollect(IntPtr arg) { - Collect(); + DisposeAll(); _pending = false; return 0; } diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 4b4d8b107..52438b386 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -21,9 +21,6 @@ public PythonException() { IntPtr gs = PythonEngine.AcquireLock(); Runtime.PyErr_Fetch(ref _pyType, ref _pyValue, ref _pyTB); - Runtime.XIncref(_pyType); - Runtime.XIncref(_pyValue); - Runtime.XIncref(_pyTB); if (_pyType != IntPtr.Zero && _pyValue != IntPtr.Zero) { string type; From ef832a33435f7d1ff77a394bee794e035b2e548e Mon Sep 17 00:00:00 2001 From: amos402 Date: Tue, 10 Jul 2018 15:15:50 +0800 Subject: [PATCH 42/76] Fixed nPython.exe crash on Shutdown --- src/runtime/finalizer.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index 2f4549256..ba1064387 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -80,6 +80,11 @@ internal void AddFinalizedObject(IDisposable obj) internal static void Shutdown() { + if (Runtime.Py_IsInitialized() == 0) + { + Instance._objQueue = new ConcurrentQueue(); + return; + } Instance.DisposeAll(); Instance.CallPendingFinalizers(); Runtime.PyErr_Clear(); From 3a9be30e31f1e343a1e6283fb85fd55a06a2d679 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sat, 4 Aug 2018 16:19:09 +0800 Subject: [PATCH 43/76] Add error handler Synchronized fixed --- src/embed_tests/TestFinalizer.cs | 54 ++++++++++++++++++++++++++++++++ src/runtime/finalizer.cs | 41 ++++++++++++++++++------ src/runtime/runtime.cs | 4 +-- 3 files changed, 87 insertions(+), 12 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 985ce3b35..f8d8b6548 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -151,5 +151,59 @@ public void SimpleTestMemory() Finalizer.Instance.Enable = oldState; } } + + class MyPyObject : PyObject + { + public MyPyObject(IntPtr op) : base(op) + { + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + GC.SuppressFinalize(this); + throw new Exception("MyPyObject"); + } + internal static void CreateMyPyObject(IntPtr op) + { + Runtime.Runtime.XIncref(op); + new MyPyObject(op); + } + } + + [Test] + public void ErrorHandling() + { + bool called = false; + EventHandler handleFunc = (sender, args) => + { + called = true; + Assert.AreEqual(args.Error.Message, "MyPyObject"); + }; + Finalizer.Instance.Threshold = 1; + Finalizer.Instance.ErrorHandler += handleFunc; + try + { + WeakReference shortWeak; + WeakReference longWeak; + { + MakeAGarbage(out shortWeak, out longWeak); + var obj = (PyLong)longWeak.Target; + IntPtr handle = obj.Handle; + shortWeak = null; + longWeak = null; + MyPyObject.CreateMyPyObject(handle); + obj.Dispose(); + obj = null; + } + FullGCCollect(); + Finalizer.Instance.Collect(); + Assert.IsTrue(called); + } + finally + { + Finalizer.Instance.ErrorHandler -= handleFunc; + } + } } } diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index ba1064387..0fc36768f 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -14,15 +14,21 @@ public class CollectArgs : EventArgs public int ObjectCount { get; set; } } + public class ErrorArgs : EventArgs + { + public Exception Error { get; set; } + } + public static readonly Finalizer Instance = new Finalizer(); public event EventHandler CollectOnce; + public event EventHandler ErrorHandler; private ConcurrentQueue _objQueue = new ConcurrentQueue(); [UnmanagedFunctionPointer(CallingConvention.Cdecl)] - private delegate int PedingCall(IntPtr arg); - private readonly PedingCall _collectAction; + private delegate int PendingCall(IntPtr arg); + private readonly PendingCall _collectAction; private bool _pending = false; private readonly object _collectingLock = new object(); @@ -87,11 +93,14 @@ internal static void Shutdown() } Instance.DisposeAll(); Instance.CallPendingFinalizers(); - Runtime.PyErr_Clear(); } private void AddPendingCollect() { + if (_pending) + { + return; + } lock (_collectingLock) { if (_pending) @@ -99,12 +108,12 @@ private void AddPendingCollect() return; } _pending = true; - } - IntPtr func = Marshal.GetFunctionPointerForDelegate(_collectAction); - if (Runtime.Py_AddPendingCall(func, IntPtr.Zero) != 0) - { - // Full queue, append next time - _pending = false; + IntPtr func = Marshal.GetFunctionPointerForDelegate(_collectAction); + if (Runtime.Py_AddPendingCall(func, IntPtr.Zero) != 0) + { + // Full queue, append next time + _pending = false; + } } } @@ -124,7 +133,19 @@ private void DisposeAll() IDisposable obj; while (_objQueue.TryDequeue(out obj)) { - obj.Dispose(); + try + { + obj.Dispose(); + Runtime.CheckExceptionOccurred(); + } + catch (Exception e) + { + // We should not bother the main thread + ErrorHandler?.Invoke(this, new ErrorArgs() + { + Error = e + }); + } } } } diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 85346790d..9ff83753f 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -199,7 +199,7 @@ public class Runtime internal static bool IsPython2 = pyversionnumber < 30; internal static bool IsPython3 = pyversionnumber >= 30; - public static int MainManagedThreadId { get; internal set; } + public static int MainManagedThreadId { get; private set; } /// /// Encoding to use to convert Unicode to/from Managed to Native @@ -362,10 +362,10 @@ internal static void Initialize() internal static void Shutdown() { - Finalizer.Shutdown(); AssemblyManager.Shutdown(); Exceptions.Shutdown(); ImportHook.Shutdown(); + Finalizer.Shutdown(); Py_Finalize(); } From eb569f2db9bcd9ee68545dad68ad7a6208119fe1 Mon Sep 17 00:00:00 2001 From: amos402 Date: Thu, 6 Sep 2018 15:06:36 +0800 Subject: [PATCH 44/76] Make collect callback without JIT --- src/runtime/finalizer.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index 0fc36768f..e6efab7c5 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -117,10 +117,10 @@ private void AddPendingCollect() } } - private int OnPendingCollect(IntPtr arg) + private static int OnPendingCollect(IntPtr arg) { - DisposeAll(); - _pending = false; + Instance.DisposeAll(); + Instance._pending = false; return 0; } From 994449ec1b8fde61b0768331e973a1741c66c690 Mon Sep 17 00:00:00 2001 From: amos402 Date: Wed, 24 Oct 2018 03:53:09 +0800 Subject: [PATCH 45/76] Add pending marker --- src/runtime/finalizer.cs | 66 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 6 deletions(-) diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index e6efab7c5..9164d68a1 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Runtime.InteropServices; using System.Threading; @@ -24,16 +25,23 @@ public class ErrorArgs : EventArgs public event EventHandler CollectOnce; public event EventHandler ErrorHandler; - private ConcurrentQueue _objQueue = new ConcurrentQueue(); + public int Threshold { get; set; } + public bool Enable { get; set; } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + struct PendingArgs + { + public bool cancelled; + } [UnmanagedFunctionPointer(CallingConvention.Cdecl)] private delegate int PendingCall(IntPtr arg); private readonly PendingCall _collectAction; + private ConcurrentQueue _objQueue = new ConcurrentQueue(); private bool _pending = false; private readonly object _collectingLock = new object(); - public int Threshold { get; set; } - public bool Enable { get; set; } + private IntPtr _pendingArgs; private Finalizer() { @@ -92,6 +100,23 @@ internal static void Shutdown() return; } Instance.DisposeAll(); + if (Thread.CurrentThread.ManagedThreadId != Runtime.MainManagedThreadId) + { + if (Instance._pendingArgs == IntPtr.Zero) + { + Instance.ResetPending(); + return; + } + // Not in main thread just cancel the pending operation to avoid error in different domain + // It will make a memory leak + unsafe + { + PendingArgs* args = (PendingArgs*)Instance._pendingArgs; + args->cancelled = true; + } + Instance.ResetPending(); + return; + } Instance.CallPendingFinalizers(); } @@ -108,8 +133,12 @@ private void AddPendingCollect() return; } _pending = true; + var args = new PendingArgs() { cancelled = false }; + IntPtr p = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(PendingArgs))); + Marshal.StructureToPtr(args, p, false); + _pendingArgs = p; IntPtr func = Marshal.GetFunctionPointerForDelegate(_collectAction); - if (Runtime.Py_AddPendingCall(func, IntPtr.Zero) != 0) + if (Runtime.Py_AddPendingCall(func, p) != 0) { // Full queue, append next time _pending = false; @@ -119,8 +148,24 @@ private void AddPendingCollect() private static int OnPendingCollect(IntPtr arg) { - Instance.DisposeAll(); - Instance._pending = false; + Debug.Assert(arg == Instance._pendingArgs); + try + { + unsafe + { + PendingArgs* pendingArgs = (PendingArgs*)arg; + if (pendingArgs->cancelled) + { + return 0; + } + } + Instance.DisposeAll(); + } + finally + { + Instance.ResetPending(); + Marshal.FreeHGlobal(arg); + } return 0; } @@ -148,5 +193,14 @@ private void DisposeAll() } } } + + private void ResetPending() + { + lock (_collectingLock) + { + _pending = false; + _pendingArgs = IntPtr.Zero; + } + } } } From fc53d947d0cace0f72e6bf6af8f65c891646d852 Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 26 Oct 2018 03:09:37 +0800 Subject: [PATCH 46/76] Remove PYTHONMALLOC setting --- src/embed_tests/TestFinalizer.cs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index f8d8b6548..3f09668a4 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -14,15 +14,6 @@ public class TestFinalizer [SetUp] public void SetUp() { - try - { - _PYTHONMALLOC = Environment.GetEnvironmentVariable("PYTHONMALLOC"); - } - catch (ArgumentNullException) - { - _PYTHONMALLOC = null; - } - Environment.SetEnvironmentVariable("PYTHONMALLOC", "malloc"); _oldThreshold = Finalizer.Instance.Threshold; PythonEngine.Initialize(); Exceptions.Clear(); @@ -33,7 +24,6 @@ public void TearDown() { Finalizer.Instance.Threshold = _oldThreshold; PythonEngine.Shutdown(); - Environment.SetEnvironmentVariable("PYTHONMALLOC", _PYTHONMALLOC); } private static void FullGCCollect() From 8eb35914efad5607a8701887bf92e49c47644a1e Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 23 Nov 2018 01:52:05 +0800 Subject: [PATCH 47/76] Fix ref count error --- src/embed_tests/TestPyAnsiString.cs | 1 + src/embed_tests/TestPyFloat.cs | 1 + src/embed_tests/TestPyString.cs | 1 + 3 files changed, 3 insertions(+) diff --git a/src/embed_tests/TestPyAnsiString.cs b/src/embed_tests/TestPyAnsiString.cs index 9ba7d6cc6..b4a965ff7 100644 --- a/src/embed_tests/TestPyAnsiString.cs +++ b/src/embed_tests/TestPyAnsiString.cs @@ -63,6 +63,7 @@ public void TestCtorPtr() const string expected = "foo"; var t = new PyAnsiString(expected); + Runtime.Runtime.XIncref(t.Handle); var actual = new PyAnsiString(t.Handle); Assert.AreEqual(expected, actual.ToString()); diff --git a/src/embed_tests/TestPyFloat.cs b/src/embed_tests/TestPyFloat.cs index f2c85a77f..94e7026c7 100644 --- a/src/embed_tests/TestPyFloat.cs +++ b/src/embed_tests/TestPyFloat.cs @@ -25,6 +25,7 @@ public void Dispose() public void IntPtrCtor() { var i = new PyFloat(1); + Runtime.Runtime.XIncref(i.Handle); var ii = new PyFloat(i.Handle); Assert.AreEqual(i.Handle, ii.Handle); } diff --git a/src/embed_tests/TestPyString.cs b/src/embed_tests/TestPyString.cs index 9d1cdb0e9..0de436e35 100644 --- a/src/embed_tests/TestPyString.cs +++ b/src/embed_tests/TestPyString.cs @@ -64,6 +64,7 @@ public void TestCtorPtr() const string expected = "foo"; var t = new PyString(expected); + Runtime.Runtime.XIncref(t.Handle); var actual = new PyString(t.Handle); Assert.AreEqual(expected, actual.ToString()); From 361022034989257bd85fca22cbca694d1900bd52 Mon Sep 17 00:00:00 2001 From: amos402 Date: Fri, 23 Nov 2018 03:37:18 +0800 Subject: [PATCH 48/76] Add ref count check for helping discover the bugs of decref too much --- src/embed_tests/TestFinalizer.cs | 42 ++++++- src/runtime/Python.Runtime.15.csproj | 8 +- src/runtime/delegatemanager.cs | 7 +- src/runtime/finalizer.cs | 157 ++++++++++++++++++++++++--- src/runtime/pyobject.cs | 30 ++++- src/runtime/pyscope.cs | 7 +- src/runtime/pythonexception.cs | 7 +- 7 files changed, 233 insertions(+), 25 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 3f09668a4..1b7faf084 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -8,7 +8,6 @@ namespace Python.EmbeddingTest { public class TestFinalizer { - private string _PYTHONMALLOC = string.Empty; private int _oldThreshold; [SetUp] @@ -195,5 +194,46 @@ public void ErrorHandling() Finalizer.Instance.ErrorHandler -= handleFunc; } } + + [Test] + public void ValidateRefCount() + { + if (!Finalizer.Instance.RefCountValidationEnabled) + { + Assert.Pass("Only run with FINALIZER_CHECK"); + } + IntPtr ptr = IntPtr.Zero; + bool called = false; + Finalizer.IncorrectRefCntHandler handler = (s, e) => + { + called = true; + Assert.AreEqual(ptr, e.Handle); + Assert.AreEqual(2, e.ImpactedObjects.Count); + // Fix for this test, don't do this on general environment + Runtime.Runtime.XIncref(e.Handle); + return false; + }; + Finalizer.Instance.IncorrectRefCntResovler += handler; + try + { + ptr = CreateStringGarbage(); + FullGCCollect(); + Assert.Throws(() => Finalizer.Instance.Collect()); + Assert.IsTrue(called); + } + finally + { + Finalizer.Instance.IncorrectRefCntResovler -= handler; + } + } + + private static IntPtr CreateStringGarbage() + { + PyString s1 = new PyString("test_string"); + // s2 steal a reference from s1 + PyString s2 = new PyString(s1.Handle); + return s1.Handle; + } + } } diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj index cfde0a127..2497844bb 100644 --- a/src/runtime/Python.Runtime.15.csproj +++ b/src/runtime/Python.Runtime.15.csproj @@ -68,10 +68,10 @@ $(DefineConstants);PYTHON3;$(Python3Version);$(PythonMonoDefineConstants) - $(DefineConstants);PYTHON2;$(Python2Version);$(PythonMonoDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON2;$(Python2Version);$(PythonMonoDefineConstants);FINALIZER_CHECK;TRACE;DEBUG - $(DefineConstants);PYTHON3;$(Python3Version);$(PythonMonoDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON3;$(Python3Version);$(PythonMonoDefineConstants);FINALIZER_CHECK;TRACE;DEBUG $(DefineConstants);PYTHON2;$(Python2Version);$(PythonWinDefineConstants) @@ -80,10 +80,10 @@ $(DefineConstants);PYTHON3;$(Python3Version);$(PythonWinDefineConstants) - $(DefineConstants);PYTHON2;$(Python2Version);$(PythonWinDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON2;$(Python2Version);$(PythonWinDefineConstants);FINALIZER_CHECK;TRACE;DEBUG - $(DefineConstants);PYTHON3;$(Python3Version);$(PythonWinDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON3;$(Python3Version);$(PythonWinDefineConstants);FINALIZER_CHECK;TRACE;DEBUG diff --git a/src/runtime/delegatemanager.cs b/src/runtime/delegatemanager.cs index 706bd4bc4..bd8f1ee4c 100644 --- a/src/runtime/delegatemanager.cs +++ b/src/runtime/delegatemanager.cs @@ -181,7 +181,7 @@ A possible alternate strategy would be to create custom subclasses too "special" for this to work. It would be more work, so for now the 80/20 rule applies :) */ - public class Dispatcher : IDisposable + public class Dispatcher : IPyDisposable { public IntPtr target; public Type dtype; @@ -276,6 +276,11 @@ public object TrueDispatch(ArrayList args) Runtime.XDecref(op); return result; } + + public IntPtr[] GetTrackedHandles() + { + return new IntPtr[] { target }; + } } diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index 9164d68a1..80519845a 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -38,11 +38,48 @@ struct PendingArgs private delegate int PendingCall(IntPtr arg); private readonly PendingCall _collectAction; - private ConcurrentQueue _objQueue = new ConcurrentQueue(); + private ConcurrentQueue _objQueue = new ConcurrentQueue(); private bool _pending = false; private readonly object _collectingLock = new object(); private IntPtr _pendingArgs; + #region FINALIZER_CHECK + +#if FINALIZER_CHECK + private readonly object _queueLock = new object(); + public bool RefCountValidationEnabled { get; set; } = true; +#else + public readonly bool RefCountValidationEnabled = false; +#endif + // Keep these declarations for compat even no FINALIZER_CHECK + public class IncorrectFinalizeArgs : EventArgs + { + public IntPtr Handle { get; internal set; } + public ICollection ImpactedObjects { get; internal set; } + } + + public class IncorrectRefCountException : Exception + { + public IntPtr PyPtr { get; internal set; } + private string _message; + public override string Message => _message; + + public IncorrectRefCountException(IntPtr ptr) + { + PyPtr = ptr; + IntPtr pyname = Runtime.PyObject_Unicode(PyPtr); + string name = Runtime.GetManagedString(pyname); + Runtime.XDecref(pyname); + _message = $"{name} may has a incorrect ref count"; + } + } + + public delegate bool IncorrectRefCntHandler(object sender, IncorrectFinalizeArgs e); + public event IncorrectRefCntHandler IncorrectRefCntResovler; + public bool ThrowIfUnhandleIncorrectRefCount { get; set; } = true; + + #endregion + private Finalizer() { Enable = true; @@ -72,7 +109,7 @@ public List GetCollectedObjects() return _objQueue.Select(T => new WeakReference(T)).ToList(); } - internal void AddFinalizedObject(IDisposable obj) + internal void AddFinalizedObject(IPyDisposable obj) { if (!Enable) { @@ -84,7 +121,12 @@ internal void AddFinalizedObject(IDisposable obj) // for avoiding that case, user should call GC.Collect manual before shutdown. return; } - _objQueue.Enqueue(obj); +#if FINALIZER_CHECK + lock (_queueLock) +#endif + { + _objQueue.Enqueue(obj); + } GC.ReRegisterForFinalize(obj); if (_objQueue.Count >= Threshold) { @@ -96,7 +138,7 @@ internal static void Shutdown() { if (Runtime.Py_IsInitialized() == 0) { - Instance._objQueue = new ConcurrentQueue(); + Instance._objQueue = new ConcurrentQueue(); return; } Instance.DisposeAll(); @@ -175,21 +217,29 @@ private void DisposeAll() { ObjectCount = _objQueue.Count }); - IDisposable obj; - while (_objQueue.TryDequeue(out obj)) +#if FINALIZER_CHECK + lock (_queueLock) +#endif { - try - { - obj.Dispose(); - Runtime.CheckExceptionOccurred(); - } - catch (Exception e) +#if FINALIZER_CHECK + ValidateRefCount(); +#endif + IPyDisposable obj; + while (_objQueue.TryDequeue(out obj)) { - // We should not bother the main thread - ErrorHandler?.Invoke(this, new ErrorArgs() + try + { + obj.Dispose(); + Runtime.CheckExceptionOccurred(); + } + catch (Exception e) { - Error = e - }); + // We should not bother the main thread + ErrorHandler?.Invoke(this, new ErrorArgs() + { + Error = e + }); + } } } } @@ -202,5 +252,80 @@ private void ResetPending() _pendingArgs = IntPtr.Zero; } } + +#if FINALIZER_CHECK + private void ValidateRefCount() + { + if (!RefCountValidationEnabled) + { + return; + } + var counter = new Dictionary(); + var holdRefs = new Dictionary(); + var indexer = new Dictionary>(); + foreach (var obj in _objQueue) + { + IntPtr[] handles = obj.GetTrackedHandles(); + foreach (var handle in handles) + { + if (handle == IntPtr.Zero) + { + continue; + } + if (!counter.ContainsKey(handle)) + { + counter[handle] = 0; + } + counter[handle]++; + if (!holdRefs.ContainsKey(handle)) + { + holdRefs[handle] = Runtime.Refcount(handle); + } + List objs; + if (!indexer.TryGetValue(handle, out objs)) + { + objs = new List(); + indexer.Add(handle, objs); + } + objs.Add(obj); + } + } + foreach (var pair in counter) + { + IntPtr handle = pair.Key; + long cnt = pair.Value; + // Tracked handle's ref count is larger than the object's holds + // it may take an unspecified behaviour if it decref in Dispose + if (cnt > holdRefs[handle]) + { + var args = new IncorrectFinalizeArgs() + { + Handle = handle, + ImpactedObjects = indexer[handle] + }; + bool handled = false; + if (IncorrectRefCntResovler != null) + { + var funcList = IncorrectRefCntResovler.GetInvocationList(); + foreach (IncorrectRefCntHandler func in funcList) + { + if (func(this, args)) + { + handled = true; + break; + } + } + } + if (!handled && ThrowIfUnhandleIncorrectRefCount) + { + throw new IncorrectRefCountException(handle); + } + } + // Make sure no other references for PyObjects after this method + indexer[handle].Clear(); + } + indexer.Clear(); + } +#endif } } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 22f6d3e88..42ba1ff39 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -1,11 +1,17 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Dynamic; using System.Linq.Expressions; namespace Python.Runtime { + public interface IPyDisposable : IDisposable + { + IntPtr[] GetTrackedHandles(); + } + /// /// Represents a generic Python object. The methods of this class are /// generally equivalent to the Python "abstract object API". See @@ -13,8 +19,15 @@ namespace Python.Runtime /// PY3: https://docs.python.org/3/c-api/object.html /// for details. /// - public class PyObject : DynamicObject, IEnumerable, IDisposable + public class PyObject : DynamicObject, IEnumerable, IPyDisposable { +#if TRACE_ALLOC + /// + /// Trace stack for PyObject's construction + /// + public StackTrace Traceback { get; private set; } +#endif + protected internal IntPtr obj = IntPtr.Zero; private bool disposed = false; private bool _finalized = false; @@ -31,6 +44,9 @@ public class PyObject : DynamicObject, IEnumerable, IDisposable public PyObject(IntPtr ptr) { obj = ptr; +#if TRACE_ALLOC + Traceback = new StackTrace(1); +#endif } // Protected default constructor to allow subclasses to manage @@ -38,12 +54,19 @@ public PyObject(IntPtr ptr) protected PyObject() { +#if TRACE_ALLOC + Traceback = new StackTrace(1); +#endif } // Ensure that encapsulated Python object is decref'ed appropriately // when the managed wrapper is garbage-collected. ~PyObject() { + if (obj == IntPtr.Zero) + { + return; + } if (_finalized || disposed) { return; @@ -176,6 +199,11 @@ public void UnsafeDispose() GC.SuppressFinalize(this); } + public IntPtr[] GetTrackedHandles() + { + return new IntPtr[] { obj }; + } + /// /// GetPythonType Method /// diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 32d9626bd..f5449f30c 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -22,7 +22,7 @@ public class PyGILAttribute : Attribute } [PyGIL] - public class PyScope : DynamicObject, IDisposable + public class PyScope : DynamicObject, IPyDisposable { public readonly string Name; @@ -526,6 +526,11 @@ public void Dispose() this.OnDispose?.Invoke(this); } + public IntPtr[] GetTrackedHandles() + { + return new IntPtr[] { obj }; + } + ~PyScope() { if (_finalized || _isDisposed) diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 52438b386..295a63b3d 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -6,7 +6,7 @@ namespace Python.Runtime /// Provides a managed interface to exceptions thrown by the Python /// runtime. /// - public class PythonException : System.Exception, IDisposable + public class PythonException : System.Exception, IPyDisposable { private IntPtr _pyType = IntPtr.Zero; private IntPtr _pyValue = IntPtr.Zero; @@ -174,6 +174,11 @@ public void Dispose() } } + public IntPtr[] GetTrackedHandles() + { + return new IntPtr[] { _pyType, _pyValue, _pyTB }; + } + /// /// Matches Method /// From e4fd72e64212049f02666072ee6f628404029452 Mon Sep 17 00:00:00 2001 From: amos402 Date: Sun, 25 Nov 2018 23:25:26 +0800 Subject: [PATCH 49/76] Fix ref count error --- src/embed_tests/TestPyInt.cs | 2 ++ src/embed_tests/TestPyLong.cs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/embed_tests/TestPyInt.cs b/src/embed_tests/TestPyInt.cs index 4117336d8..005ab466d 100644 --- a/src/embed_tests/TestPyInt.cs +++ b/src/embed_tests/TestPyInt.cs @@ -86,6 +86,7 @@ public void TestCtorSByte() public void TestCtorPtr() { var i = new PyInt(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyInt(i.Handle); Assert.AreEqual(5, a.ToInt32()); } @@ -94,6 +95,7 @@ public void TestCtorPtr() public void TestCtorPyObject() { var i = new PyInt(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyInt(i); Assert.AreEqual(5, a.ToInt32()); } diff --git a/src/embed_tests/TestPyLong.cs b/src/embed_tests/TestPyLong.cs index fe3e13ef5..3c155f315 100644 --- a/src/embed_tests/TestPyLong.cs +++ b/src/embed_tests/TestPyLong.cs @@ -102,6 +102,7 @@ public void TestCtorDouble() public void TestCtorPtr() { var i = new PyLong(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyLong(i.Handle); Assert.AreEqual(5, a.ToInt32()); } @@ -110,6 +111,7 @@ public void TestCtorPtr() public void TestCtorPyObject() { var i = new PyLong(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyLong(i); Assert.AreEqual(5, a.ToInt32()); } From 0ed8bee39f3b0715e8fe7fb96bf161ce9ec00f4d Mon Sep 17 00:00:00 2001 From: amos402 Date: Mon, 26 Nov 2018 17:45:11 +0800 Subject: [PATCH 50/76] typo error --- src/embed_tests/TestFinalizer.cs | 4 ++-- src/runtime/finalizer.cs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 1b7faf084..bb90c92cf 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -213,7 +213,7 @@ public void ValidateRefCount() Runtime.Runtime.XIncref(e.Handle); return false; }; - Finalizer.Instance.IncorrectRefCntResovler += handler; + Finalizer.Instance.IncorrectRefCntResolver += handler; try { ptr = CreateStringGarbage(); @@ -223,7 +223,7 @@ public void ValidateRefCount() } finally { - Finalizer.Instance.IncorrectRefCntResovler -= handler; + Finalizer.Instance.IncorrectRefCntResolver -= handler; } } diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index 80519845a..a94f7be31 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -75,7 +75,7 @@ public IncorrectRefCountException(IntPtr ptr) } public delegate bool IncorrectRefCntHandler(object sender, IncorrectFinalizeArgs e); - public event IncorrectRefCntHandler IncorrectRefCntResovler; + public event IncorrectRefCntHandler IncorrectRefCntResolver; public bool ThrowIfUnhandleIncorrectRefCount { get; set; } = true; #endregion @@ -304,9 +304,9 @@ private void ValidateRefCount() ImpactedObjects = indexer[handle] }; bool handled = false; - if (IncorrectRefCntResovler != null) + if (IncorrectRefCntResolver != null) { - var funcList = IncorrectRefCntResovler.GetInvocationList(); + var funcList = IncorrectRefCntResolver.GetInvocationList(); foreach (IncorrectRefCntHandler func in funcList) { if (func(this, args)) From 2c94c9040c3f59e8cb48c9fbf7ee1b63db68c672 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Wed, 27 Mar 2019 17:43:16 -0300 Subject: [PATCH 51/76] Version bump 1.0.5.18 --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 4be598ac8..fe2774f48 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.17 +current_version = 1.0.5.18 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 1e85fffac..20d1d67f5 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.17" + version: "1.0.5.18" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 42230e099..5ed496e1f 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.17", + version="1.0.5.18", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index ba4cfde67..845ff8f0d 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.17")] +[assembly: AssemblyVersion("1.0.5.18")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 2d08d31ee..6403a0563 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.17"), + Version = new Version("1.0.5.18"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 279d7f4f2..610186ecb 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.17" +__version__ = "1.0.5.18" class clrproperty(object): From 56a8e2eeb126c13e25982d66d5f33b8ec50d5efd Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Tue, 2 Apr 2019 12:01:52 -0300 Subject: [PATCH 52/76] Fix memory leak in finalizer - `finalizer` was leaking `_pendingArgs` (global memory) when the call to `Py_AddPendingCall` was unsuccessful. - Changing `lock` for `Monitor.TryEnter` at `AddPendingCollect()` so threads don't block each other. --- src/runtime/finalizer.cs | 50 ++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index a94f7be31..bab301af8 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -41,7 +41,7 @@ struct PendingArgs private ConcurrentQueue _objQueue = new ConcurrentQueue(); private bool _pending = false; private readonly object _collectingLock = new object(); - private IntPtr _pendingArgs; + private IntPtr _pendingArgs = IntPtr.Zero; #region FINALIZER_CHECK @@ -128,7 +128,7 @@ internal void AddFinalizedObject(IPyDisposable obj) _objQueue.Enqueue(obj); } GC.ReRegisterForFinalize(obj); - if (_objQueue.Count >= Threshold) + if (!_pending && _objQueue.Count >= Threshold) { AddPendingCollect(); } @@ -164,26 +164,28 @@ internal static void Shutdown() private void AddPendingCollect() { - if (_pending) + if(Monitor.TryEnter(_collectingLock)) { - return; - } - lock (_collectingLock) - { - if (_pending) + try { - return; + if (!_pending) + { + _pending = true; + var args = new PendingArgs { cancelled = false }; + _pendingArgs = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(PendingArgs))); + Marshal.StructureToPtr(args, _pendingArgs, false); + IntPtr func = Marshal.GetFunctionPointerForDelegate(_collectAction); + if (Runtime.Py_AddPendingCall(func, _pendingArgs) != 0) + { + // Full queue, append next time + FreePendingArgs(); + _pending = false; + } + } } - _pending = true; - var args = new PendingArgs() { cancelled = false }; - IntPtr p = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(PendingArgs))); - Marshal.StructureToPtr(args, p, false); - _pendingArgs = p; - IntPtr func = Marshal.GetFunctionPointerForDelegate(_collectAction); - if (Runtime.Py_AddPendingCall(func, p) != 0) + finally { - // Full queue, append next time - _pending = false; + Monitor.Exit(_collectingLock); } } } @@ -205,8 +207,8 @@ private static int OnPendingCollect(IntPtr arg) } finally { + Instance.FreePendingArgs(); Instance.ResetPending(); - Marshal.FreeHGlobal(arg); } return 0; } @@ -244,12 +246,20 @@ private void DisposeAll() } } + private void FreePendingArgs() + { + if (_pendingArgs != IntPtr.Zero) + { + Marshal.FreeHGlobal(_pendingArgs); + _pendingArgs = IntPtr.Zero; + } + } + private void ResetPending() { lock (_collectingLock) { _pending = false; - _pendingArgs = IntPtr.Zero; } } From c9421947160f3fa2b1836ff75be99663bf1a4c2d Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Tue, 2 Apr 2019 12:07:25 -0300 Subject: [PATCH 53/76] Version bump 1.0.5.19 --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index fe2774f48..816220243 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.18 +current_version = 1.0.5.19 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 20d1d67f5..74fb214f9 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.18" + version: "1.0.5.19" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 5ed496e1f..72d3f9f01 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.18", + version="1.0.5.19", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index 845ff8f0d..891ef7fc6 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.18")] +[assembly: AssemblyVersion("1.0.5.19")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 6403a0563..336446cb7 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.18"), + Version = new Version("1.0.5.19"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 610186ecb..6e97ba637 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.18" +__version__ = "1.0.5.19" class clrproperty(object): From 758470883d0c53c83cbbece53747ba10594b27cd Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Tue, 23 Apr 2019 16:19:26 -0300 Subject: [PATCH 54/76] Fix memory leak - finalizer - The callback set by `Runtime.Py_AddPendingCall()` was not being triggered in some cases in a multithreading environment. Replacing it with a `Task` --- src/embed_tests/TestFinalizer.cs | 12 +-- src/runtime/finalizer.cs | 136 +++++++++---------------------- 2 files changed, 44 insertions(+), 104 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index bb90c92cf..dd578becc 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -36,13 +36,12 @@ public void CollectBasicObject() { Assert.IsTrue(Finalizer.Instance.Enable); - int thId = Thread.CurrentThread.ManagedThreadId; Finalizer.Instance.Threshold = 1; bool called = false; + var objectCount = 0; EventHandler handler = (s, e) => { - Assert.AreEqual(thId, Thread.CurrentThread.ManagedThreadId); - Assert.GreaterOrEqual(e.ObjectCount, 1); + objectCount = e.ObjectCount; called = true; }; @@ -73,6 +72,7 @@ public void CollectBasicObject() Finalizer.Instance.CollectOnce -= handler; } Assert.IsTrue(called); + Assert.GreaterOrEqual(objectCount, 1); } private static void MakeAGarbage(out WeakReference shortWeak, out WeakReference longWeak) @@ -85,7 +85,7 @@ private static void MakeAGarbage(out WeakReference shortWeak, out WeakReference private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) { - // Must larger than 512 bytes make sure Python use + // Must larger than 512 bytes make sure Python use string str = new string('1', 1024); Finalizer.Instance.Enable = true; FullGCCollect(); @@ -164,10 +164,11 @@ internal static void CreateMyPyObject(IntPtr op) public void ErrorHandling() { bool called = false; + var errorMessage = ""; EventHandler handleFunc = (sender, args) => { called = true; - Assert.AreEqual(args.Error.Message, "MyPyObject"); + errorMessage = args.Error.Message; }; Finalizer.Instance.Threshold = 1; Finalizer.Instance.ErrorHandler += handleFunc; @@ -193,6 +194,7 @@ public void ErrorHandling() { Finalizer.Instance.ErrorHandler -= handleFunc; } + Assert.AreEqual(errorMessage, "MyPyObject"); } [Test] diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index bab301af8..dd519e227 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -1,10 +1,9 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; -using System.Runtime.InteropServices; using System.Threading; +using System.Threading.Tasks; namespace Python.Runtime { @@ -28,20 +27,10 @@ public class ErrorArgs : EventArgs public int Threshold { get; set; } public bool Enable { get; set; } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - struct PendingArgs - { - public bool cancelled; - } - - [UnmanagedFunctionPointer(CallingConvention.Cdecl)] - private delegate int PendingCall(IntPtr arg); - private readonly PendingCall _collectAction; - private ConcurrentQueue _objQueue = new ConcurrentQueue(); private bool _pending = false; private readonly object _collectingLock = new object(); - private IntPtr _pendingArgs = IntPtr.Zero; + private Task _finalizerTask; #region FINALIZER_CHECK @@ -84,23 +73,26 @@ private Finalizer() { Enable = true; Threshold = 200; - _collectAction = OnPendingCollect; } - public void CallPendingFinalizers() + public bool CallPendingFinalizers() { - if (Thread.CurrentThread.ManagedThreadId != Runtime.MainManagedThreadId) + if (Instance._finalizerTask != null + && !Instance._finalizerTask.IsCompleted) { - throw new Exception("PendingCall should execute in main Python thread"); + var ts = PythonEngine.BeginAllowThreads(); + Instance._finalizerTask.Wait(); + PythonEngine.EndAllowThreads(ts); + return true; } - Runtime.Py_MakePendingCalls(); + return false; } public void Collect() { - using (var gilState = new Py.GILState()) + if (!Instance.CallPendingFinalizers()) { - DisposeAll(); + Instance.DisposeAll(); } } @@ -141,25 +133,10 @@ internal static void Shutdown() Instance._objQueue = new ConcurrentQueue(); return; } - Instance.DisposeAll(); - if (Thread.CurrentThread.ManagedThreadId != Runtime.MainManagedThreadId) + if(!Instance.CallPendingFinalizers()) { - if (Instance._pendingArgs == IntPtr.Zero) - { - Instance.ResetPending(); - return; - } - // Not in main thread just cancel the pending operation to avoid error in different domain - // It will make a memory leak - unsafe - { - PendingArgs* args = (PendingArgs*)Instance._pendingArgs; - args->cancelled = true; - } - Instance.ResetPending(); - return; + Instance.DisposeAll(); } - Instance.CallPendingFinalizers(); } private void AddPendingCollect() @@ -171,16 +148,14 @@ private void AddPendingCollect() if (!_pending) { _pending = true; - var args = new PendingArgs { cancelled = false }; - _pendingArgs = Marshal.AllocHGlobal(Marshal.SizeOf(typeof(PendingArgs))); - Marshal.StructureToPtr(args, _pendingArgs, false); - IntPtr func = Marshal.GetFunctionPointerForDelegate(_collectAction); - if (Runtime.Py_AddPendingCall(func, _pendingArgs) != 0) + // should already be complete but just in case + _finalizerTask?.Wait(); + + _finalizerTask = Task.Factory.StartNew(() => { - // Full queue, append next time - FreePendingArgs(); + Instance.DisposeAll(); _pending = false; - } + }); } } finally @@ -190,29 +165,6 @@ private void AddPendingCollect() } } - private static int OnPendingCollect(IntPtr arg) - { - Debug.Assert(arg == Instance._pendingArgs); - try - { - unsafe - { - PendingArgs* pendingArgs = (PendingArgs*)arg; - if (pendingArgs->cancelled) - { - return 0; - } - } - Instance.DisposeAll(); - } - finally - { - Instance.FreePendingArgs(); - Instance.ResetPending(); - } - return 0; - } - private void DisposeAll() { CollectOnce?.Invoke(this, new CollectArgs() @@ -223,46 +175,32 @@ private void DisposeAll() lock (_queueLock) #endif { + using (Py.GIL()) + { #if FINALIZER_CHECK - ValidateRefCount(); + ValidateRefCount(); #endif - IPyDisposable obj; - while (_objQueue.TryDequeue(out obj)) - { - try + IPyDisposable obj; + while (_objQueue.TryDequeue(out obj)) { - obj.Dispose(); - Runtime.CheckExceptionOccurred(); - } - catch (Exception e) - { - // We should not bother the main thread - ErrorHandler?.Invoke(this, new ErrorArgs() + try { - Error = e - }); + obj.Dispose(); + Runtime.CheckExceptionOccurred(); + } + catch (Exception e) + { + // We should not bother the main thread + ErrorHandler?.Invoke(this, new ErrorArgs() + { + Error = e + }); + } } } } } - private void FreePendingArgs() - { - if (_pendingArgs != IntPtr.Zero) - { - Marshal.FreeHGlobal(_pendingArgs); - _pendingArgs = IntPtr.Zero; - } - } - - private void ResetPending() - { - lock (_collectingLock) - { - _pending = false; - } - } - #if FINALIZER_CHECK private void ValidateRefCount() { From a30321c9d6a7d3a285a8eec4eec6770f1c7b6ef0 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Tue, 23 Apr 2019 17:33:15 -0300 Subject: [PATCH 55/76] Version bump 1.0.5.20 --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 816220243..9fa0e95b6 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.19 +current_version = 1.0.5.20 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 74fb214f9..4bf1ac6a3 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.19" + version: "1.0.5.20" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 72d3f9f01..d5107eca9 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.19", + version="1.0.5.20", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index 891ef7fc6..a31d80dd8 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.19")] +[assembly: AssemblyVersion("1.0.5.20")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 336446cb7..a6dbdcc65 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.19"), + Version = new Version("1.0.5.20"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 6e97ba637..4d7487511 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.19" +__version__ = "1.0.5.20" class clrproperty(object): From b1e84442ff76d13b1864071908281604aa2497ae Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Tue, 23 Apr 2019 18:33:41 -0300 Subject: [PATCH 56/76] Fix unit test - Refactor --- src/embed_tests/TestFinalizer.cs | 8 ++++---- src/runtime/finalizer.cs | 24 +++++++++--------------- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index dd578becc..53838f315 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -45,6 +45,9 @@ public void CollectBasicObject() called = true; }; + Assert.IsFalse(called); + Finalizer.Instance.CollectOnce += handler; + WeakReference shortWeak; WeakReference longWeak; { @@ -60,12 +63,9 @@ public void CollectBasicObject() Assert.NotZero(garbage.Count); Assert.IsTrue(garbage.Any(T => ReferenceEquals(T.Target, longWeak.Target))); } - - Assert.IsFalse(called); - Finalizer.Instance.CollectOnce += handler; try { - Finalizer.Instance.CallPendingFinalizers(); + Finalizer.Instance.Collect(forceDispose: false); } finally { diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index dd519e227..948f94cb5 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -75,22 +75,19 @@ private Finalizer() Threshold = 200; } - public bool CallPendingFinalizers() + public void Collect(bool forceDispose = true) { if (Instance._finalizerTask != null && !Instance._finalizerTask.IsCompleted) { - var ts = PythonEngine.BeginAllowThreads(); - Instance._finalizerTask.Wait(); - PythonEngine.EndAllowThreads(ts); - return true; + using (Py.GIL()) + { + var ts = PythonEngine.BeginAllowThreads(); + Instance._finalizerTask.Wait(); + PythonEngine.EndAllowThreads(ts); + } } - return false; - } - - public void Collect() - { - if (!Instance.CallPendingFinalizers()) + else if (forceDispose) { Instance.DisposeAll(); } @@ -133,10 +130,7 @@ internal static void Shutdown() Instance._objQueue = new ConcurrentQueue(); return; } - if(!Instance.CallPendingFinalizers()) - { - Instance.DisposeAll(); - } + Instance.Collect(forceDispose: true); } private void AddPendingCollect() From bf368fb342c5f70ea638a90926f348c2f8468dc5 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Mon, 29 Apr 2019 19:27:31 -0300 Subject: [PATCH 57/76] Fix deadlock when shuting down --- src/runtime/finalizer.cs | 47 +++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index 948f94cb5..dd5c0b4dd 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -80,12 +80,9 @@ public void Collect(bool forceDispose = true) if (Instance._finalizerTask != null && !Instance._finalizerTask.IsCompleted) { - using (Py.GIL()) - { - var ts = PythonEngine.BeginAllowThreads(); - Instance._finalizerTask.Wait(); - PythonEngine.EndAllowThreads(ts); - } + var ts = PythonEngine.BeginAllowThreads(); + Instance._finalizerTask.Wait(); + PythonEngine.EndAllowThreads(ts); } else if (forceDispose) { @@ -147,8 +144,11 @@ private void AddPendingCollect() _finalizerTask = Task.Factory.StartNew(() => { - Instance.DisposeAll(); - _pending = false; + using (Py.GIL()) + { + Instance.DisposeAll(); + _pending = false; + } }); } } @@ -169,27 +169,24 @@ private void DisposeAll() lock (_queueLock) #endif { - using (Py.GIL()) - { #if FINALIZER_CHECK - ValidateRefCount(); + ValidateRefCount(); #endif - IPyDisposable obj; - while (_objQueue.TryDequeue(out obj)) + IPyDisposable obj; + while (_objQueue.TryDequeue(out obj)) + { + try { - try - { - obj.Dispose(); - Runtime.CheckExceptionOccurred(); - } - catch (Exception e) + obj.Dispose(); + Runtime.CheckExceptionOccurred(); + } + catch (Exception e) + { + // We should not bother the main thread + ErrorHandler?.Invoke(this, new ErrorArgs() { - // We should not bother the main thread - ErrorHandler?.Invoke(this, new ErrorArgs() - { - Error = e - }); - } + Error = e + }); } } } From c70724a30c74722bca767ba1294fd6275f8a3290 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Fri, 14 Jun 2019 13:47:28 -0700 Subject: [PATCH 58/76] Improve performance of unwrapping .NET objects passed from Python This addresses the following scenario: 1. A C# object `a` is created and filled with some data. 2. `a` is passed via Python.NET to Python. To do that Python.NET creates a wrapper object `w`, and stores reference to `a` in one of its fields. 3. Python code later passes `w` back to C#, e.g. calls `SomeCSharpMethod(w)`. 4. Python.NET has to unwrap `w`, so it reads the reference to `a` from it. Prior to this change in 4. Python.NET had to determine what kind of an object `a` is. If it is an exception, a different offset needed to be used. That check was very expensive (up to 4 calls into Python interpreter). This change replaces that check with computing offset unconditionally by subtracting a constant from the object size (which is read from the wrapper), thus avoiding calls to Python interpreter. --- AUTHORS.md | 1 + CHANGELOG.md | 1 + src/runtime/classbase.cs | 2 +- src/runtime/classderived.cs | 4 +- src/runtime/clrobject.cs | 4 +- src/runtime/interop.cs | 123 +++++++++++++++++++++++++----------- src/runtime/managedtype.cs | 2 +- src/runtime/moduleobject.cs | 2 +- src/runtime/typemanager.cs | 8 +-- 9 files changed, 100 insertions(+), 47 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index 3c39794e4..fc0fdf86b 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -39,6 +39,7 @@ - Sam Winstanley ([@swinstanley](https://github.com/swinstanley)) - Sean Freitag ([@cowboygneox](https://github.com/cowboygneox)) - Serge Weinstock ([@sweinst](https://github.com/sweinst)) +- Victor Milovanov([@lostmsu](https://github.com/lostmsu)) - Ville M. Vainio ([@vivainio](https://github.com/vivainio)) - Virgil Dupras ([@hsoft](https://github.com/hsoft)) - Wenguang Yang ([@yagweb](https://github.com/yagweb)) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9db05d81..b92703d43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Changed - PythonException included C# call stack +- improved performance of calls from Python to C# ### Fixed diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 4dd3b5364..128e9795e 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -271,7 +271,7 @@ public static int tp_is_gc(IntPtr type) public static void tp_dealloc(IntPtr ob) { ManagedType self = GetManagedObject(ob); - IntPtr dict = Marshal.ReadIntPtr(ob, ObjectOffset.DictOffset(ob)); + IntPtr dict = Marshal.ReadIntPtr(ob, ObjectOffset.TypeDictOffset(self.tpHandle)); if (dict != IntPtr.Zero) { Runtime.XDecref(dict); diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index 16d3b99db..8ae96695f 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Reflection; @@ -870,7 +870,7 @@ public static void Finalize(IPythonDerivedType obj) // the C# object is being destroyed which must mean there are no more // references to the Python object as well so now we can dealloc the // python object. - IntPtr dict = Marshal.ReadIntPtr(self.pyHandle, ObjectOffset.DictOffset(self.pyHandle)); + IntPtr dict = Marshal.ReadIntPtr(self.pyHandle, ObjectOffset.TypeDictOffset(self.tpHandle)); if (dict != IntPtr.Zero) { Runtime.XDecref(dict); diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 17782f026..a7a160aa3 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -14,11 +14,11 @@ internal CLRObject(object ob, IntPtr tp) long flags = Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Subclass) != 0) { - IntPtr dict = Marshal.ReadIntPtr(py, ObjectOffset.DictOffset(tp)); + IntPtr dict = Marshal.ReadIntPtr(py, ObjectOffset.TypeDictOffset(tp)); if (dict == IntPtr.Zero) { dict = Runtime.PyDict_New(); - Marshal.WriteIntPtr(py, ObjectOffset.DictOffset(tp), dict); + Marshal.WriteIntPtr(py, ObjectOffset.TypeDictOffset(tp), dict); } } diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 5168bcd98..9b4ab7f57 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -1,6 +1,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.InteropServices; using System.Reflection; using System.Text; @@ -67,11 +68,47 @@ public ModulePropertyAttribute() } } + internal static class ManagedDataOffsets + { + static ManagedDataOffsets() + { + FieldInfo[] fi = typeof(ManagedDataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); + for (int i = 0; i < fi.Length; i++) + { + fi[i].SetValue(null, -(i * IntPtr.Size) - IntPtr.Size); + } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class ObjectOffset + size = fi.Length * IntPtr.Size; + } + + public static readonly int ob_data; + public static readonly int ob_dict; + + private static int BaseOffset(IntPtr type) + { + Debug.Assert(type != IntPtr.Zero); + int typeSize = Marshal.ReadInt32(type, TypeOffset.tp_basicsize); + Debug.Assert(typeSize > 0 && typeSize <= ExceptionOffset.Size()); + return typeSize; + } + public static int DataOffset(IntPtr type) + { + return BaseOffset(type) + ob_data; + } + + public static int DictOffset(IntPtr type) + { + return BaseOffset(type) + ob_dict; + } + + public static int Size { get { return size; } } + + private static readonly int size; + } + + internal static class OriginalObjectOffsets { - static ObjectOffset() + static OriginalObjectOffsets() { int size = IntPtr.Size; var n = 0; // Py_TRACE_REFS add two pointers to PyObject_HEAD @@ -82,39 +119,58 @@ static ObjectOffset() #endif ob_refcnt = (n + 0) * size; ob_type = (n + 1) * size; - ob_dict = (n + 2) * size; - ob_data = (n + 3) * size; } - public static int magic(IntPtr ob) + public static int Size { get { return size; } } + + private static readonly int size = +#if PYTHON_WITH_PYDEBUG + 4 * IntPtr.Size; +#else + 2 * IntPtr.Size; +#endif + +#if PYTHON_WITH_PYDEBUG + public static int _ob_next; + public static int _ob_prev; +#endif + public static int ob_refcnt; + public static int ob_type; + } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal class ObjectOffset + { + static ObjectOffset() + { +#if PYTHON_WITH_PYDEBUG + _ob_next = OriginalObjectOffsets._ob_next; + _ob_prev = OriginalObjectOffsets._ob_prev; +#endif + ob_refcnt = OriginalObjectOffsets.ob_refcnt; + ob_type = OriginalObjectOffsets.ob_type; + + size = OriginalObjectOffsets.Size + ManagedDataOffsets.Size; + } + + public static int magic(IntPtr type) { - if (IsException(ob)) - { - return ExceptionOffset.ob_data; - } - return ob_data; + return ManagedDataOffsets.DataOffset(type); } - public static int DictOffset(IntPtr ob) + public static int TypeDictOffset(IntPtr type) { - if (IsException(ob)) - { - return ExceptionOffset.ob_dict; - } - return ob_dict; + return ManagedDataOffsets.DictOffset(type); } - public static int Size(IntPtr ob) + public static int Size(IntPtr pyType) { - if (IsException(ob)) + if (IsException(pyType)) { return ExceptionOffset.Size(); } -#if PYTHON_WITH_PYDEBUG - return 6 * IntPtr.Size; -#else - return 4 * IntPtr.Size; -#endif + + return size; } #if PYTHON_WITH_PYDEBUG @@ -123,8 +179,7 @@ public static int Size(IntPtr ob) #endif public static int ob_refcnt; public static int ob_type; - private static int ob_dict; - private static int ob_data; + private static readonly int size; private static bool IsException(IntPtr pyObject) { @@ -141,19 +196,17 @@ internal class ExceptionOffset static ExceptionOffset() { Type type = typeof(ExceptionOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; + FieldInfo[] fi = type.GetFields(BindingFlags.Static | BindingFlags.Public); for (int i = 0; i < fi.Length; i++) { - fi[i].SetValue(null, (i * size) + ObjectOffset.ob_type + size); + fi[i].SetValue(null, (i * IntPtr.Size) + OriginalObjectOffsets.Size); } - } - public static int Size() - { - return ob_data + IntPtr.Size; + size = fi.Length * IntPtr.Size + OriginalObjectOffsets.Size + ManagedDataOffsets.Size; } + public static int Size() { return size; } + // PyException_HEAD // (start after PyObject_HEAD) public static int dict = 0; @@ -167,9 +220,7 @@ public static int Size() public static int suppress_context = 0; #endif - // extra c# data - public static int ob_dict; - public static int ob_data; + private static readonly int size; } diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 3191da949..23f5898d1 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -33,7 +33,7 @@ internal static ManagedType GetManagedObject(IntPtr ob) { IntPtr op = tp == ob ? Marshal.ReadIntPtr(tp, TypeOffset.magic()) - : Marshal.ReadIntPtr(ob, ObjectOffset.magic(ob)); + : Marshal.ReadIntPtr(ob, ObjectOffset.magic(tp)); if (op == IntPtr.Zero) { return null; diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 4accb1531..c06eaf4f0 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -53,7 +53,7 @@ public ModuleObject(string name) Runtime.XDecref(pyfilename); Runtime.XDecref(pydocstring); - Marshal.WriteIntPtr(pyHandle, ObjectOffset.DictOffset(pyHandle), dict); + Marshal.WriteIntPtr(pyHandle, ObjectOffset.TypeDictOffset(tpHandle), dict); InitializeModuleMembers(); } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 6570ee083..9b8f95649 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections; using System.Collections.Generic; using System.Reflection; @@ -79,7 +79,7 @@ internal static IntPtr CreateType(Type impl) // Set tp_basicsize to the size of our managed instance objects. Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); - var offset = (IntPtr)ObjectOffset.DictOffset(type); + var offset = (IntPtr)ObjectOffset.TypeDictOffset(type); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); InitializeSlots(type, impl); @@ -117,7 +117,6 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) IntPtr base_ = IntPtr.Zero; int ob_size = ObjectOffset.Size(Runtime.PyTypeType); - int tp_dictoffset = ObjectOffset.DictOffset(Runtime.PyTypeType); // XXX Hack, use a different base class for System.Exception // Python 2.5+ allows new style class exceptions but they *must* @@ -125,9 +124,10 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) if (typeof(Exception).IsAssignableFrom(clrType)) { ob_size = ObjectOffset.Size(Exceptions.Exception); - tp_dictoffset = ObjectOffset.DictOffset(Exceptions.Exception); } + int tp_dictoffset = ob_size + ManagedDataOffsets.ob_dict; + if (clrType == typeof(Exception)) { base_ = Exceptions.Exception; From e07a74eac0fa52c78bf8fc2791d019dbab34a2da Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 24 Jun 2019 14:00:11 -0500 Subject: [PATCH 59/76] Version bump to 1-0-5-21 --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 9fa0e95b6..2629b1d38 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.20 +current_version = 1.0.5.21 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 4bf1ac6a3..aef4c8f5c 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.20" + version: "1.0.5.21" build: skip: True # [not win] diff --git a/setup.py b/setup.py index d5107eca9..f945c7948 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.20", + version="1.0.5.21", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index a31d80dd8..9be7c6ce2 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.20")] +[assembly: AssemblyVersion("1.0.5.21")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index a6dbdcc65..16e1b0fd9 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.20"), + Version = new Version("1.0.5.21"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 4d7487511..ba1323792 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.20" +__version__ = "1.0.5.21" class clrproperty(object): From 37ddac6158709f15bf015ae56eb6cd651d690bb5 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Mon, 26 Aug 2019 17:35:07 -0300 Subject: [PATCH 60/76] Version bump to 1-0-5-22 --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 2629b1d38..020f3e885 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.21 +current_version = 1.0.5.22 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index aef4c8f5c..9a15f79f3 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.21" + version: "1.0.5.22" build: skip: True # [not win] diff --git a/setup.py b/setup.py index f945c7948..c6e5500db 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.21", + version="1.0.5.22", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index 9be7c6ce2..b199b93b7 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.21")] +[assembly: AssemblyVersion("1.0.5.22")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 16e1b0fd9..ff4d85c27 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.21"), + Version = new Version("1.0.5.22"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index ba1323792..53a2c62be 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.21" +__version__ = "1.0.5.22" class clrproperty(object): From 695d569d5342bf8cb594033c26a4542b86c45ab3 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Tue, 3 Sep 2019 17:19:27 -0300 Subject: [PATCH 61/76] Add initialization logs --- src/runtime/pythonengine.cs | 6 +++++- src/runtime/runtime.cs | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index a23c7ac79..5a62c02bb 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -164,6 +164,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true) // during an initial "import clr", and the world ends shortly thereafter. // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). delegateManager = new DelegateManager(); + Console.WriteLine("PythonEngine.Initialize(): Runtime.Initialize()..."); Runtime.Initialize(); initialized = true; Exceptions.Clear(); @@ -179,9 +180,11 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true) string code = "import atexit, clr\n" + "atexit.register(clr._AtExit)\n"; + Console.WriteLine("PythonEngine.Initialize(): register atexit callback..."); PythonEngine.Exec(code); // Load the clr.py resource into the clr module + Console.WriteLine("PythonEngine.Initialize(): GetCLRModule()..."); IntPtr clr = Python.Runtime.ImportHook.GetCLRModule(); IntPtr clr_dict = Runtime.PyModule_GetDict(clr); @@ -193,6 +196,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true) IntPtr builtins = Runtime.PyEval_GetBuiltins(); Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); + Console.WriteLine("PythonEngine.Initialize(): clr GetManifestResourceStream..."); Assembly assembly = Assembly.GetExecutingAssembly(); using (Stream stream = assembly.GetManifestResourceStream("clr.py")) using (var reader = new StreamReader(stream)) diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 9ff83753f..06740273a 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -213,15 +213,18 @@ internal static void Initialize() { if (Py_IsInitialized() == 0) { + Console.WriteLine("Runtime.Initialize(): Py_Initialize..."); Py_Initialize(); MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; } if (PyEval_ThreadsInitialized() == 0) { + Console.WriteLine("Runtime.Initialize(): PyEval_InitThreads..."); PyEval_InitThreads(); } + Console.WriteLine("Runtime.Initialize(): Initialize types..."); IntPtr op; IntPtr dict; if (IsPython3) @@ -326,6 +329,7 @@ internal static void Initialize() XDecref(c); XDecref(d); #endif + Console.WriteLine("Runtime.Initialize(): Initialize types end."); Error = new IntPtr(-1); @@ -343,7 +347,7 @@ internal static void Initialize() NativeMethods.FreeLibrary(dllLocal); } #endif - + Console.WriteLine("Runtime.Initialize(): AssemblyManager.Initialize()..."); // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); PyCLRMetaType = MetaType.Initialize(); @@ -357,6 +361,7 @@ internal static void Initialize() IntPtr item = PyString_FromString(rtdir); PyList_Append(path, item); XDecref(item); + Console.WriteLine("Runtime.Initialize(): AssemblyManager.UpdatePath()..."); AssemblyManager.UpdatePath(); } From db20fd767c9bf614cccd62d70f5c313432f10f27 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Thu, 5 Sep 2019 16:47:38 -0300 Subject: [PATCH 62/76] Version bump 1.0.5.23 --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 020f3e885..3a5888143 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.22 +current_version = 1.0.5.23 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 9a15f79f3..cb1c123a2 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.22" + version: "1.0.5.23" build: skip: True # [not win] diff --git a/setup.py b/setup.py index c6e5500db..f55479725 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.22", + version="1.0.5.23", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index b199b93b7..99a1c71f2 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.22")] +[assembly: AssemblyVersion("1.0.5.23")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index ff4d85c27..fc99f9fcd 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.22"), + Version = new Version("1.0.5.23"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 53a2c62be..a4f998ab1 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.22" +__version__ = "1.0.5.23" class clrproperty(object): From b9242d2c9bd61b5b0bc30473debf9144f4bd583b Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Tue, 10 Sep 2019 11:37:36 -0300 Subject: [PATCH 63/76] Fix initialization hang. Version 1.0.5.24 --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/assemblymanager.cs | 6 ++++++ src/runtime/resources/clr.py | 2 +- 7 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 3a5888143..5aef0f399 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.23 +current_version = 1.0.5.24 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index cb1c123a2..7eddf826b 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.23" + version: "1.0.5.24" build: skip: True # [not win] diff --git a/setup.py b/setup.py index f55479725..4b0bbb89d 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.23", + version="1.0.5.24", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index 99a1c71f2..e08d13878 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.23")] +[assembly: AssemblyVersion("1.0.5.24")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index fc99f9fcd..8d9cc71d2 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.23"), + Version = new Version("1.0.5.24"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index d63930a58..4f0f1e417 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -344,6 +344,12 @@ internal static void ScanAssembly(Assembly assembly) // gather a list of all of the namespaces contributed to by // the assembly. + // skip this assembly, it causes 'GetTypes' call to hang + if (assembly.FullName.StartsWith("System.Windows.Forms")) + { + return; + } + Type[] types = assembly.GetTypes(); foreach (Type t in types) { diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index a4f998ab1..7b431ed74 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.23" +__version__ = "1.0.5.24" class clrproperty(object): From b98085cda54fa083630cb3deccaaf8b3be7bb232 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Mon, 23 Sep 2019 17:03:36 -0300 Subject: [PATCH 64/76] Fix implicit conversion method resolution - Fixes for implicit conversions which did not work and fix for methodBinder method resolution, will prioritize methods that do not required implicit conversions - Adding unit tests --- src/embed_tests/Python.EmbeddingTest.csproj | 1 + src/embed_tests/TestMethodBinder.cs | 151 +++++++++++++++ src/runtime/classobject.cs | 4 +- src/runtime/constructorbinding.cs | 4 +- src/runtime/converter.cs | 11 ++ src/runtime/indexer.cs | 14 +- src/runtime/methodbinder.cs | 204 +++++++++++++------- src/runtime/methodobject.cs | 6 +- 8 files changed, 307 insertions(+), 88 deletions(-) create mode 100644 src/embed_tests/TestMethodBinder.cs diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index 11ef2daac..b0aa650d3 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -88,6 +88,7 @@ + diff --git a/src/embed_tests/TestMethodBinder.cs b/src/embed_tests/TestMethodBinder.cs new file mode 100644 index 000000000..ba4096776 --- /dev/null +++ b/src/embed_tests/TestMethodBinder.cs @@ -0,0 +1,151 @@ +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestMethodBinder + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void ImplicitConversionToString() + { + // create instance of python model + dynamic pyMethodCall = PythonEngine.ModuleFromString("module", @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") + +from Python.EmbeddingTest import * + +class PythonModel(TestMethodBinder.CSharpModel): + def MethodCall(self): + return self.OnlyString(TestMethodBinder.TestImplicitConversion()) +").GetAttr("PythonModel").Invoke(); + + using (Py.GIL()) + { + var data = (string)pyMethodCall.MethodCall(); + // we assert implicit conversion took place + Assert.AreEqual("OnlyString impl: implicit to string", data); + } + } + + [Test] + public void ImplicitConversionToClass() + { + // create instance of python model + dynamic pyMethodCall = PythonEngine.ModuleFromString("module", @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") + +from Python.EmbeddingTest import * + +class PythonModel(TestMethodBinder.CSharpModel): + def MethodCall(self): + return self.OnlyClass('input string') +").GetAttr("PythonModel").Invoke(); + + using (Py.GIL()) + { + var data = (string)pyMethodCall.MethodCall(); + // we assert implicit conversion took place + Assert.AreEqual("OnlyClass impl", data); + } + } + + [Test] + public void WillAvoidUsingImplicitConversionIfPossible_String() + { + // create instance of python model + dynamic pyMethodCall = PythonEngine.ModuleFromString("module", @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") + +from Python.EmbeddingTest import * + +class PythonModel(TestMethodBinder.CSharpModel): + def MethodCall(self): + return self.InvokeModel('input string') +").GetAttr("PythonModel").Invoke(); + + using (Py.GIL()) + { + var data = (string)pyMethodCall.MethodCall(); + // we assert no implicit conversion took place + Assert.AreEqual("string impl: input string", data); + } + } + + [Test] + public void WillAvoidUsingImplicitConversionIfPossible_Class() + { + // create instance of python model + dynamic pyMethodCall = PythonEngine.ModuleFromString("module", @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") + +from Python.EmbeddingTest import * + +class PythonModel(TestMethodBinder.CSharpModel): + def MethodCall(self): + return self.InvokeModel(TestMethodBinder.TestImplicitConversion()) +").GetAttr("PythonModel").Invoke(); + + using (Py.GIL()) + { + var data = (string)pyMethodCall.MethodCall(); + // we assert no implicit conversion took place + Assert.AreEqual("TestImplicitConversion impl", data); + } + } + + public class CSharpModel + { + public virtual string OnlyClass(TestImplicitConversion data) + { + return "OnlyClass impl"; + } + + public virtual string OnlyString(string data) + { + return "OnlyString impl: " + data; + } + + public virtual string InvokeModel(string data) + { + return "string impl: " + data; + } + + public virtual string InvokeModel(TestImplicitConversion data) + { + return "TestImplicitConversion impl"; + } + } + + public class TestImplicitConversion + { + public static implicit operator string(TestImplicitConversion symbol) + { + return "implicit to string"; + } + public static implicit operator TestImplicitConversion(string symbol) + { + return new TestImplicitConversion(); + } + } + } +} diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs index 46257c73f..ef8316c76 100644 --- a/src/runtime/classobject.cs +++ b/src/runtime/classobject.cs @@ -31,9 +31,9 @@ internal ClassObject(Type tp) : base(tp) /// internal IntPtr GetDocString() { - MethodBase[] methods = binder.GetMethods(); + var methods = binder.GetMethods(); var str = ""; - foreach (MethodBase t in methods) + foreach (var t in methods) { if (str.Length > 0) { diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 3908628b9..de7284031 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -119,10 +119,10 @@ public static IntPtr tp_repr(IntPtr ob) Runtime.XIncref(self.repr); return self.repr; } - MethodBase[] methods = self.ctorBinder.GetMethods(); + var methods = self.ctorBinder.GetMethods(); string name = self.type.FullName; var doc = ""; - foreach (MethodBase t in methods) + foreach (var t in methods) { if (doc.Length > 0) { diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index a81d9fb99..7beeb8759 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -397,6 +397,17 @@ internal static bool ToManagedValue(IntPtr value, Type obType, result = tmp; return true; } + else + { + var type = tmp.GetType(); + // check implicit conversions that receive tmp type and return obType + var conversionMethod = type.GetMethod("op_Implicit", new[] { type }); + if (conversionMethod != null && conversionMethod.ReturnType == obType) + { + result = conversionMethod.Invoke(null, new[] { tmp }); + return true; + } + } Exceptions.SetError(Exceptions.TypeError, $"value cannot be converted to {obType}"); return false; } diff --git a/src/runtime/indexer.cs b/src/runtime/indexer.cs index 7b6d90ca8..6379ba968 100644 --- a/src/runtime/indexer.cs +++ b/src/runtime/indexer.cs @@ -56,15 +56,14 @@ internal void SetItem(IntPtr inst, IntPtr args) internal bool NeedsDefaultArgs(IntPtr args) { - int pynargs = Runtime.PyTuple_Size(args); - MethodBase[] methods = SetterBinder.GetMethods(); - if (methods.Length == 0) + var methods = SetterBinder.GetMethods(); + if (methods.Count == 0) { return false; } + int pynargs = Runtime.PyTuple_Size(args); - MethodBase mi = methods[0]; - ParameterInfo[] pi = mi.GetParameters(); + var pi = methods[0].ParameterInfo; // need to subtract one for the value int clrnargs = pi.Length - 1; if (pynargs == clrnargs || pynargs > clrnargs) @@ -98,9 +97,8 @@ internal IntPtr GetDefaultArgs(IntPtr args) int pynargs = Runtime.PyTuple_Size(args); // Get the default arg tuple - MethodBase[] methods = SetterBinder.GetMethods(); - MethodBase mi = methods[0]; - ParameterInfo[] pi = mi.GetParameters(); + var methods = SetterBinder.GetMethods(); + ParameterInfo[] pi = methods[0].ParameterInfo; int clrnargs = pi.Length - 1; IntPtr defaultArgs = Runtime.PyTuple_New(clrnargs - pynargs); for (var i = 0; i < clrnargs - pynargs; i++) diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 48dc1eb9a..b73f84569 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -1,5 +1,6 @@ using System; using System.Collections; +using System.Collections.Generic; using System.Reflection; namespace Python.Runtime @@ -12,19 +13,18 @@ namespace Python.Runtime /// internal class MethodBinder { - public ArrayList list; - public MethodBase[] methods; + private List list; public bool init = false; public bool allow_threads = true; internal MethodBinder() { - list = new ArrayList(); + list = new List(); } internal MethodBinder(MethodInfo mi) { - list = new ArrayList { mi }; + list = new List { new MethodInformation(mi, mi.GetParameters()) }; } public int Count @@ -34,7 +34,9 @@ public int Count internal void AddMethod(MethodBase m) { - list.Add(m); + // we added a new method so we have to re sort the method list + init = false; + list.Add(new MethodInformation(m, m.GetParameters())); } /// @@ -154,16 +156,15 @@ internal static MethodInfo MatchSignatureAndParameters(MethodInfo[] mi, Type[] g /// is arranged in order of precedence (done lazily to avoid doing it /// at all for methods that are never called). /// - internal MethodBase[] GetMethods() + internal List GetMethods() { if (!init) { // I'm sure this could be made more efficient. list.Sort(new MethodSorter()); - methods = (MethodBase[])list.ToArray(typeof(MethodBase)); init = true; } - return methods; + return list; } /// @@ -174,9 +175,10 @@ internal MethodBase[] GetMethods() /// Based from Jython `org.python.core.ReflectedArgs.precedence` /// See: https://github.com/jythontools/jython/blob/master/src/org/python/core/ReflectedArgs.java#L192 /// - internal static int GetPrecedence(MethodBase mi) + private static int GetPrecedence(MethodInformation methodInformation) { - ParameterInfo[] pi = mi.GetParameters(); + ParameterInfo[] pi = methodInformation.ParameterInfo; + var mi = methodInformation.MethodBase; int val = mi.IsStatic ? 3000 : 0; int num = pi.Length; @@ -290,65 +292,38 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, MethodInfo[] methodinfo) { // loop to find match, return invoker w/ or /wo error - MethodBase[] _methods = null; int pynargs = Runtime.PyTuple_Size(args); object arg; var isGeneric = false; - ArrayList defaultArgList = null; - if (info != null) - { - _methods = new MethodBase[1]; - _methods.SetValue(info, 0); - } - else - { - _methods = GetMethods(); - } + ArrayList defaultArgList; Type clrtype; + Binding bindingUsingImplicitConversion = null; + + var methods = info == null ? GetMethods() + : new List(1) { new MethodInformation(info, info.GetParameters()) }; + // TODO: Clean up - foreach (MethodBase mi in _methods) + foreach (var methodInformation in methods) { + var mi = methodInformation.MethodBase; + var pi = methodInformation.ParameterInfo; + if (mi.IsGenericMethod) { isGeneric = true; } - ParameterInfo[] pi = mi.GetParameters(); int clrnargs = pi.Length; - var match = false; - int arrayStart = -1; - var outs = 0; + int arrayStart; - if (pynargs == clrnargs) - { - match = true; - } - else if (pynargs < clrnargs) - { - match = true; - defaultArgList = new ArrayList(); - for (int v = pynargs; v < clrnargs; v++) - { - if (pi[v].DefaultValue == DBNull.Value) - { - match = false; - } - else - { - defaultArgList.Add(pi[v].DefaultValue); - } - } - } - else if (pynargs > clrnargs && clrnargs > 0 && - Attribute.IsDefined(pi[clrnargs - 1], typeof(ParamArrayAttribute))) - { - // This is a `foo(params object[] bar)` style method - match = true; - arrayStart = clrnargs - 1; - } - - if (match) + if (CheckMethodArgumentsMatch(clrnargs, + pynargs, + pi, + out arrayStart, + out defaultArgList)) { + var outs = 0; var margs = new object[clrnargs]; + var usedImplicitConversion = false; for (var n = 0; n < clrnargs; n++) { @@ -371,7 +346,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth // is necessary clrtype = null; IntPtr pyoptype; - if (_methods.Length > 1) + if (methods.Count > 1) { pyoptype = IntPtr.Zero; pyoptype = Runtime.PyObject_Type(op); @@ -430,7 +405,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth var opImplicit = pi[n].ParameterType.GetMethod("op_Implicit", new[] { clrtype }); if (opImplicit != null) { - typematch = opImplicit.ReturnType == pi[n].ParameterType; + usedImplicitConversion = typematch = opImplicit.ReturnType == pi[n].ParameterType; clrtype = pi[n].ParameterType; } } @@ -504,9 +479,31 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth target = co.inst; } - return new Binding(mi, target, margs, outs); + var binding = new Binding(mi, target, margs, outs); + if (usedImplicitConversion) + { + // lets just keep the first binding using implicit conversion + // this is to respect method order/precedence + if (bindingUsingImplicitConversion == null) + { + // in this case we will not return the binding yet in case there is a match + // which does not use implicit conversions, which will return directly + bindingUsingImplicitConversion = binding; + } + } + else + { + return binding; + } } } + + // if we generated a binding using implicit conversion return it + if (bindingUsingImplicitConversion != null) + { + return bindingUsingImplicitConversion; + } + // We weren't able to find a matching method but at least one // is a generic method and info is null. That happens when a generic // method was not called using the [] syntax. Let's introspect the @@ -520,6 +517,47 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth return null; } + private bool CheckMethodArgumentsMatch(int clrnargs, + int pynargs, + ParameterInfo[] parameterInfo, + out int arrayStart, + out ArrayList defaultArgList) + { + arrayStart = -1; + defaultArgList = null; + + var match = false; + if (pynargs == clrnargs) + { + match = true; + } + else if (pynargs < clrnargs) + { + match = true; + defaultArgList = new ArrayList(); + for (var v = pynargs; v < clrnargs && match; v++) + { + if (parameterInfo[v].DefaultValue == DBNull.Value) + { + match = false; + } + else + { + defaultArgList.Add(parameterInfo[v].DefaultValue); + } + } + } + else if (pynargs > clrnargs && clrnargs > 0 && + Attribute.IsDefined(parameterInfo[clrnargs - 1], typeof(ParamArrayAttribute))) + { + // This is a `foo(params object[] bar)` style method + match = true; + arrayStart = clrnargs - 1; + } + + return match; + } + internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw) { return Invoke(inst, args, kw, null, null); @@ -622,27 +660,47 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i return Converter.ToPython(result, mi.ReturnType); } - } - - /// - /// Utility class to sort method info by parameter type precedence. - /// - internal class MethodSorter : IComparer - { - int IComparer.Compare(object m1, object m2) + /// + /// Utility class to store the information about a + /// + internal class MethodInformation { - int p1 = MethodBinder.GetPrecedence((MethodBase)m1); - int p2 = MethodBinder.GetPrecedence((MethodBase)m2); - if (p1 < p2) + public MethodBase MethodBase { get; } + + public ParameterInfo[] ParameterInfo { get; } + + public MethodInformation(MethodBase methodBase, ParameterInfo[] parameterInfo) { - return -1; + MethodBase = methodBase; + ParameterInfo = parameterInfo; } - if (p1 > p2) + + public override string ToString() + { + return MethodBase.ToString(); + } + } + + /// + /// Utility class to sort method info by parameter type precedence. + /// + private class MethodSorter : IComparer + { + public int Compare(MethodInformation x, MethodInformation y) { - return 1; + int p1 = GetPrecedence(x); + int p2 = GetPrecedence(y); + if (p1 < p2) + { + return -1; + } + if (p1 > p2) + { + return 1; + } + return 0; } - return 0; } } diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 8df9c8029..2e67787d2 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -68,14 +68,14 @@ internal IntPtr GetDocString() } var str = ""; Type marker = typeof(DocStringAttribute); - MethodBase[] methods = binder.GetMethods(); - foreach (MethodBase method in methods) + var methods = binder.GetMethods(); + foreach (var method in methods) { if (str.Length > 0) { str += Environment.NewLine; } - var attrs = (Attribute[])method.GetCustomAttributes(marker, false); + var attrs = (Attribute[])method.MethodBase.GetCustomAttributes(marker, false); if (attrs.Length == 0) { str += method.ToString(); From a466d0daf6c0f81d3f53b6963558fc5a94a1496c Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Mon, 23 Sep 2019 18:11:25 -0300 Subject: [PATCH 65/76] Version bump 1.0.5.25 --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 5aef0f399..f4750c71e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.24 +current_version = 1.0.5.25 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 7eddf826b..d87edf13f 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.24" + version: "1.0.5.25" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 4b0bbb89d..6ed8b5d0a 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.24", + version="1.0.5.25", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index e08d13878..6b846d8d5 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.24")] +[assembly: AssemblyVersion("1.0.5.25")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 8d9cc71d2..2674e2ac8 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.24"), + Version = new Version("1.0.5.25"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 7b431ed74..2182c3267 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.24" +__version__ = "1.0.5.25" class clrproperty(object): From 053a1044b88bf176eafd4a433d27e2d29a252303 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Mon, 23 Sep 2019 19:24:47 -0300 Subject: [PATCH 66/76] Address review: add comment for new method --- src/runtime/methodbinder.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index b73f84569..854f386ab 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -517,6 +517,10 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth return null; } + /// + /// This helper method will perform an initial check to determine if we found a matching + /// method based on its parameters count and type + /// private bool CheckMethodArgumentsMatch(int clrnargs, int pynargs, ParameterInfo[] parameterInfo, From 814fc30cc0bcc233ac7de6e5238be82a91e9d836 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Tue, 15 Oct 2019 16:29:39 -0300 Subject: [PATCH 67/76] Implement PyList to C# list conversion --- src/embed_tests/TestConverter.cs | 28 +++++++++++- src/runtime/converter.cs | 76 ++++++++++++++++++++++++++------ 2 files changed, 90 insertions(+), 14 deletions(-) diff --git a/src/embed_tests/TestConverter.cs b/src/embed_tests/TestConverter.cs index 346c8afdc..0f35570de 100644 --- a/src/embed_tests/TestConverter.cs +++ b/src/embed_tests/TestConverter.cs @@ -1,4 +1,6 @@ -using NUnit.Framework; +using System; +using System.Collections.Generic; +using NUnit.Framework; using Python.Runtime; namespace Python.EmbeddingTest @@ -17,6 +19,30 @@ public void Dispose() PythonEngine.Shutdown(); } + [Test] + public void ConvertListRoundTrip() + { + var list = new List { typeof(decimal), typeof(int) }; + var py = list.ToPython(); + object result; + var converted = Converter.ToManaged(py.Handle, typeof(List), out result, false); + + Assert.IsTrue(converted); + Assert.AreEqual(result, list); + } + + [Test] + public void ConvertPyListToArray() + { + var array = new List { typeof(decimal), typeof(int) }; + var py = array.ToPython(); + object result; + var converted = Converter.ToManaged(py.Handle, typeof(Type[]), out result, false); + + Assert.IsTrue(converted); + Assert.AreEqual(result, array); + } + [Test] public void TestConvertSingleToManaged( [Values(float.PositiveInfinity, float.NegativeInfinity, float.MinValue, float.MaxValue, float.NaN, diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 7beeb8759..b394a3f46 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -1,5 +1,6 @@ using System; using System.Collections; +using System.Collections.Generic; using System.Globalization; using System.Runtime.InteropServices; using System.Security; @@ -166,19 +167,20 @@ internal static IntPtr ToPython(object value, Type type) return result; } - if (value is IList && value.GetType().IsGenericType) + var list = value as IList; + if (list != null && value.GetType().IsGenericType) { - using (var resultlist = new PyList()) + using (var resultList = new PyList()) { - foreach (object o in (IEnumerable)value) + for (var i = 0; i < list.Count; i++) { - using (var p = new PyObject(ToPython(o, o?.GetType()))) + using (var p = list[i].ToPython()) { - resultlist.Append(p); + resultList.Append(p); } } - Runtime.XIncref(resultlist.Handle); - return resultlist.Handle; + Runtime.XIncref(resultList.Handle); + return resultList.Handle; } } @@ -382,6 +384,15 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return true; } + if(obType.IsGenericType && Runtime.PyObject_TYPE(value) == Runtime.PyListType) + { + var typeDefinition = obType.GetGenericTypeDefinition(); + if (typeDefinition == typeof(List<>)) + { + return ToList(value, obType, out result, setError); + } + } + // Common case: if the Python value is a wrapped managed object // instance, just return the wrapped object. ManagedType mt = ManagedType.GetManagedObject(value); @@ -1003,7 +1014,6 @@ private static void SetConversionError(IntPtr value, Type target) Exceptions.SetError(Exceptions.TypeError, $"Cannot convert {src} to {target}"); } - /// /// Convert a Python value to a correctly typed managed array instance. /// The Python value must support the Python sequence protocol and the @@ -1015,7 +1025,7 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s int size = Runtime.PySequence_Size(value); result = null; - if (size < 0 || elementType.IsGenericType) + if (elementType.IsGenericType) { if (setError) { @@ -1026,7 +1036,49 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s Array items = Array.CreateInstance(elementType, size); - // XXX - is there a better way to unwrap this if it is a real array? + var index = 0; + result = items; + return ApplyActionToPySequence(value, obType, setError, size, elementType, o => + { + items.SetValue(o, index++); + }); + } + + /// + /// Convert a Python value to a correctly typed managed list instance. + /// The Python value must support the Python sequence protocol and the + /// items in the sequence must be convertible to the target list type. + /// + private static bool ToList(IntPtr value, Type obType, out object result, bool setError) + { + var elementType = obType.GetGenericArguments()[0]; + var size = Runtime.PySequence_Size(value); + + result = Activator.CreateInstance(obType, args: size); + var resultList = (IList)result; + return ApplyActionToPySequence(value, obType, setError, size, elementType, o => resultList.Add(o)); + } + + /// + /// Helper method that will enumerate a Python sequence convert its values to the given + /// type and send them to the provided action + /// + private static bool ApplyActionToPySequence(IntPtr value, + Type obType, + bool setError, + int size, + Type elementType, + Action action) + { + if (size < 0) + { + if (setError) + { + SetConversionError(value, obType); + } + return false; + } + for (var i = 0; i < size; i++) { object obj = null; @@ -1046,15 +1098,13 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s return false; } - items.SetValue(obj, i); + action(obj); Runtime.XDecref(item); } - result = items; return true; } - /// /// Convert a Python value to a correctly typed managed enum instance. /// From 079949ad6a9574a282f088a907cb819442864a1b Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Tue, 15 Oct 2019 16:33:52 -0300 Subject: [PATCH 68/76] Version bump 1.0.5.26 --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f4750c71e..66c2a831e 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.25 +current_version = 1.0.5.26 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index d87edf13f..b3433a871 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.25" + version: "1.0.5.26" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 6ed8b5d0a..27fc0edf3 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.25", + version="1.0.5.26", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index 6b846d8d5..96376e07a 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.25")] +[assembly: AssemblyVersion("1.0.5.26")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 2674e2ac8..d5112da67 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.25"), + Version = new Version("1.0.5.26"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 2182c3267..23aad8d32 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.25" +__version__ = "1.0.5.26" class clrproperty(object): From 0268925799418cda5f4601760a7bd40634cc504b Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Mon, 11 Nov 2019 16:25:28 -0300 Subject: [PATCH 69/76] Improve assembly manager initialization - Perform assembly scan in separate worker tasks for performance and to avoid initialization hang --- src/runtime/assemblymanager.cs | 165 ++++++++++++++++++--------------- src/runtime/classmanager.cs | 5 +- src/runtime/genericutil.cs | 141 ++++++++++++++-------------- src/runtime/moduleobject.cs | 24 +---- 4 files changed, 168 insertions(+), 167 deletions(-) diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 4f0f1e417..1beaf4789 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -1,11 +1,11 @@ using System; -using System.Collections; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Reflection; using System.Threading; +using System.Threading.Tasks; namespace Python.Runtime { @@ -17,17 +17,15 @@ internal class AssemblyManager { // modified from event handlers below, potentially triggered from different .NET threads // therefore this should be a ConcurrentDictionary - private static ConcurrentDictionary> namespaces; - //private static Dictionary> generics; - private static AssemblyLoadEventHandler lhandler; - private static ResolveEventHandler rhandler; + private static ConcurrentDictionary> namespaces; + private static ConcurrentDictionary assembliesNamesCache; + private static ConcurrentDictionary lookupTypeCache; + private static ConcurrentQueue assemblies; + private static int pendingAssemblies; // updated only under GIL? private static Dictionary probed; - - // modified from event handlers below, potentially triggered from different .NET threads - private static ConcurrentQueue assemblies; - internal static List pypath; + private static List pypath; private AssemblyManager() { @@ -40,33 +38,41 @@ private AssemblyManager() /// internal static void Initialize() { - namespaces = new ConcurrentDictionary>(); + namespaces = new ConcurrentDictionary>(); + assembliesNamesCache = new ConcurrentDictionary(); + lookupTypeCache = new ConcurrentDictionary(); probed = new Dictionary(32); - //generics = new Dictionary>(); assemblies = new ConcurrentQueue(); pypath = new List(16); AppDomain domain = AppDomain.CurrentDomain; - lhandler = new AssemblyLoadEventHandler(AssemblyLoadHandler); - domain.AssemblyLoad += lhandler; - - rhandler = new ResolveEventHandler(ResolveHandler); - domain.AssemblyResolve += rhandler; + domain.AssemblyLoad += AssemblyLoadHandler; + domain.AssemblyResolve += ResolveHandler; - Assembly[] items = domain.GetAssemblies(); - foreach (Assembly a in items) + foreach (var assembly in domain.GetAssemblies()) { try { - ScanAssembly(a); - assemblies.Enqueue(a); + LaunchAssemblyLoader(assembly); } catch (Exception ex) { - Debug.WriteLine("Error scanning assembly {0}. {1}", a, ex); + Debug.WriteLine("Error scanning assembly {0}. {1}", assembly, ex); } } + + var safeCount = 0; + // lets wait until all assemblies are loaded + do + { + if (safeCount++ > 200) + { + throw new TimeoutException("Timeout while waiting for assemblies to load"); + } + + Thread.Sleep(50); + } while (pendingAssemblies > 0); } @@ -76,8 +82,8 @@ internal static void Initialize() internal static void Shutdown() { AppDomain domain = AppDomain.CurrentDomain; - domain.AssemblyLoad -= lhandler; - domain.AssemblyResolve -= rhandler; + domain.AssemblyLoad -= AssemblyLoadHandler; + domain.AssemblyResolve -= ResolveHandler; } @@ -88,13 +94,41 @@ internal static void Shutdown() /// so that we can know about assemblies that get loaded after the /// Python runtime is initialized. /// + /// Scanning assemblies here caused internal hangs when calling + /// private static void AssemblyLoadHandler(object ob, AssemblyLoadEventArgs args) { Assembly assembly = args.LoadedAssembly; - assemblies.Enqueue(assembly); - ScanAssembly(assembly); + LaunchAssemblyLoader(assembly); } + /// + /// Launches a new task that will load the provided assembly + /// + private static void LaunchAssemblyLoader(Assembly assembly) + { + if (assembly != null) + { + Interlocked.Increment(ref pendingAssemblies); + Task.Factory.StartNew(() => + { + try + { + if (assembliesNamesCache.TryAdd(assembly.GetName().Name, assembly)) + { + assemblies.Enqueue(assembly); + ScanAssembly(assembly); + } + } + catch + { + // pass + } + + Interlocked.Decrement(ref pendingAssemblies); + }); + } + } /// /// Event handler for assembly resolve events. This is needed because @@ -106,12 +140,12 @@ private static void AssemblyLoadHandler(object ob, AssemblyLoadEventArgs args) private static Assembly ResolveHandler(object ob, ResolveEventArgs args) { string name = args.Name.ToLower(); - foreach (Assembly a in assemblies) + foreach (var assembly in assemblies) { - string full = a.FullName.ToLower(); + var full = assembly.FullName.ToLower(); if (full.StartsWith(name)) { - return a; + return assembly; } } return LoadAssemblyPath(args.Name); @@ -133,23 +167,26 @@ internal static void UpdatePath() { IntPtr list = Runtime.PySys_GetObject("path"); int count = Runtime.PyList_Size(list); + var sep = Path.DirectorySeparatorChar; + if (count != pypath.Count) { pypath.Clear(); probed.Clear(); + // add first the current path + pypath.Add(""); for (var i = 0; i < count; i++) { IntPtr item = Runtime.PyList_GetItem(list, i); string path = Runtime.GetManagedString(item); if (path != null) { - pypath.Add(path); + pypath.Add(path == string.Empty ? path : path + sep); } } } } - /// /// Given an assembly name, try to find this assembly file using the /// PYTHONPATH. If not found, return null to indicate implicit load @@ -157,30 +194,17 @@ internal static void UpdatePath() /// public static string FindAssembly(string name) { - char sep = Path.DirectorySeparatorChar; - - foreach (string head in pypath) + foreach (var head in pypath) { - string path; - if (head == null || head.Length == 0) - { - path = name; - } - else + var dll = $"{head}{name}.dll"; + if (File.Exists(dll)) { - path = head + sep + name; + return dll; } - - string temp = path + ".dll"; - if (File.Exists(temp)) - { - return temp; - } - - temp = path + ".exe"; - if (File.Exists(temp)) + var executable = $"{head}{name}.exe"; + if (File.Exists(executable)) { - return temp; + return executable; } } return null; @@ -200,10 +224,7 @@ public static Assembly LoadAssembly(string name) } catch (Exception) { - //if (!(e is System.IO.FileNotFoundException)) - //{ - // throw; - //} + // ignored } return assembly; } @@ -214,7 +235,7 @@ public static Assembly LoadAssembly(string name) /// public static Assembly LoadAssemblyPath(string name) { - string path = FindAssembly(name); + var path = FindAssembly(name); Assembly assembly = null; if (path != null) { @@ -224,6 +245,7 @@ public static Assembly LoadAssemblyPath(string name) } catch (Exception) { + // ignored } } return assembly; @@ -251,6 +273,7 @@ public static Assembly LoadAssemblyFullPath(string name) } catch (Exception) { + // ignored } } } @@ -262,14 +285,8 @@ public static Assembly LoadAssemblyFullPath(string name) /// public static Assembly FindLoadedAssembly(string name) { - foreach (Assembly a in assemblies) - { - if (a.GetName().Name == name) - { - return a; - } - } - return null; + Assembly result; + return assembliesNamesCache.TryGetValue(name, out result) ? result : null; } /// @@ -306,10 +323,6 @@ public static bool LoadImplicit(string name, bool warn = true) { a = LoadAssemblyPath(s); } - if (a == null) - { - a = LoadAssembly(s); - } if (a != null && !assembliesSet.Contains(a)) { loaded = true; @@ -344,12 +357,6 @@ internal static void ScanAssembly(Assembly assembly) // gather a list of all of the namespaces contributed to by // the assembly. - // skip this assembly, it causes 'GetTypes' call to hang - if (assembly.FullName.StartsWith("System.Windows.Forms")) - { - return; - } - Type[] types = assembly.GetTypes(); foreach (Type t in types) { @@ -361,13 +368,13 @@ internal static void ScanAssembly(Assembly assembly) for (var n = 0; n < names.Length; n++) { s = n == 0 ? names[0] : s + "." + names[n]; - namespaces.TryAdd(s, new ConcurrentDictionary()); + namespaces.TryAdd(s, new ConcurrentDictionary()); } } if (ns != null) { - namespaces[ns].TryAdd(assembly, string.Empty); + namespaces[ns].TryAdd(assembly, 1); } if (ns != null && t.IsGenericTypeDefinition) @@ -457,11 +464,17 @@ public static List GetNames(string nsname) /// public static Type LookupType(string qname) { + Type type; + if (lookupTypeCache.TryGetValue(qname, out type)) + { + return type; + } foreach (Assembly assembly in assemblies) { - Type type = assembly.GetType(qname); + type = assembly.GetType(qname); if (type != null) { + lookupTypeCache[qname] = type; return type; } } diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 6a9d40ebd..83495cec4 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -40,9 +40,8 @@ static ClassManager() /// internal static ClassBase GetClass(Type type) { - ClassBase cb = null; - cache.TryGetValue(type, out cb); - if (cb != null) + ClassBase cb; + if (cache.TryGetValue(type, out cb)) { return cb; } diff --git a/src/runtime/genericutil.cs b/src/runtime/genericutil.cs index 9772d082f..507a901c8 100644 --- a/src/runtime/genericutil.cs +++ b/src/runtime/genericutil.cs @@ -25,32 +25,33 @@ static GenericUtil() /// internal static void Register(Type t) { - if (null == t.Namespace || null == t.Name) + lock (mapping) { - return; - } + if (null == t.Namespace || null == t.Name) + { + return; + } - Dictionary> nsmap = null; - mapping.TryGetValue(t.Namespace, out nsmap); - if (nsmap == null) - { - nsmap = new Dictionary>(); - mapping[t.Namespace] = nsmap; - } - string basename = t.Name; - int tick = basename.IndexOf("`"); - if (tick > -1) - { - basename = basename.Substring(0, tick); - } - List gnames = null; - nsmap.TryGetValue(basename, out gnames); - if (gnames == null) - { - gnames = new List(); - nsmap[basename] = gnames; + Dictionary> nsmap; + if (!mapping.TryGetValue(t.Namespace, out nsmap)) + { + nsmap = new Dictionary>(); + mapping[t.Namespace] = nsmap; + } + string basename = t.Name; + int tick = basename.IndexOf("`"); + if (tick > -1) + { + basename = basename.Substring(0, tick); + } + List gnames; + if (!nsmap.TryGetValue(basename, out gnames)) + { + gnames = new List(); + nsmap[basename] = gnames; + } + gnames.Add(t.Name); } - gnames.Add(t.Name); } /// @@ -58,18 +59,20 @@ internal static void Register(Type t) /// public static List GetGenericBaseNames(string ns) { - Dictionary> nsmap = null; - mapping.TryGetValue(ns, out nsmap); - if (nsmap == null) - { - return null; - } - var names = new List(); - foreach (string key in nsmap.Keys) + lock (mapping) { - names.Add(key); + Dictionary> nsmap; + if (!mapping.TryGetValue(ns, out nsmap)) + { + return null; + } + var names = new List(); + foreach (string key in nsmap.Keys) + { + names.Add(key); + } + return names; } - return names; } /// @@ -99,38 +102,38 @@ public static List GenericsForType(Type t) public static List GenericsByName(string ns, string basename) { - Dictionary> nsmap = null; - mapping.TryGetValue(ns, out nsmap); - if (nsmap == null) + lock (mapping) { - return null; - } + Dictionary> nsmap; + if (!mapping.TryGetValue(ns, out nsmap)) + { + return null; + } - int tick = basename.IndexOf("`"); - if (tick > -1) - { - basename = basename.Substring(0, tick); - } + int tick = basename.IndexOf("`"); + if (tick > -1) + { + basename = basename.Substring(0, tick); + } - List names = null; - nsmap.TryGetValue(basename, out names); - if (names == null) - { - return null; - } + List names; + if (!nsmap.TryGetValue(basename, out names)) + { + return null; + } - var result = new List(); - foreach (string name in names) - { - string qname = ns + "." + name; - Type o = AssemblyManager.LookupType(qname); - if (o != null) + var result = new List(); + foreach (string name in names) { - result.Add(o); + string qname = ns + "." + name; + Type o = AssemblyManager.LookupType(qname); + if (o != null) + { + result.Add(o); + } } + return result; } - - return result; } /// @@ -138,17 +141,19 @@ public static List GenericsByName(string ns, string basename) /// public static string GenericNameForBaseName(string ns, string name) { - Dictionary> nsmap = null; - mapping.TryGetValue(ns, out nsmap); - if (nsmap == null) - { - return null; - } - List gnames = null; - nsmap.TryGetValue(name, out gnames); - if (gnames?.Count > 0) + lock (mapping) { - return gnames[0]; + Dictionary> nsmap; + if (!mapping.TryGetValue(ns, out nsmap)) + { + return null; + } + List gnames = null; + nsmap.TryGetValue(name, out gnames); + if (gnames?.Count > 0) + { + return gnames[0]; + } } return null; } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index c06eaf4f0..4522fb9fa 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -67,9 +67,8 @@ public ModuleObject(string name) /// public ManagedType GetAttribute(string name, bool guess) { - ManagedType cached = null; - cache.TryGetValue(name, out cached); - if (cached != null) + ManagedType cached; + if (cache.TryGetValue(name, out cached)) { return cached; } @@ -78,16 +77,6 @@ public ManagedType GetAttribute(string name, bool guess) ClassBase c; Type type; - //if (AssemblyManager.IsValidNamespace(name)) - //{ - // IntPtr py_mod_name = Runtime.PyString_FromString(name); - // IntPtr modules = Runtime.PyImport_GetModuleDict(); - // IntPtr module = Runtime.PyDict_GetItem(modules, py_mod_name); - // if (module != IntPtr.Zero) - // return (ManagedType)this; - // return null; - //} - string qname = _namespace == string.Empty ? name : _namespace + "." + name; @@ -186,14 +175,9 @@ private void StoreAttribute(string name, ManagedType ob) /// public void LoadNames() { - ManagedType m = null; - foreach (string name in AssemblyManager.GetNames(_namespace)) + foreach (var name in AssemblyManager.GetNames(_namespace)) { - cache.TryGetValue(name, out m); - if (m == null) - { - ManagedType attr = GetAttribute(name, true); - } + var attr = GetAttribute(name, true); } } From 3fef6242ed2858303eccca8967ca6a9dde09126f Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Mon, 11 Nov 2019 16:54:45 -0300 Subject: [PATCH 70/76] Version bump 1.0.5.28 --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 66c2a831e..c3f226ae6 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.26 +current_version = 1.0.5.28 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index b3433a871..c322f4a39 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.26" + version: "1.0.5.28" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 27fc0edf3..75bfb5c77 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.26", + version="1.0.5.28", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index 96376e07a..e1b69d556 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.26")] +[assembly: AssemblyVersion("1.0.5.28")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index d5112da67..89bbcf1aa 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.26"), + Version = new Version("1.0.5.28"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 23aad8d32..61b127d94 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.26" +__version__ = "1.0.5.28" class clrproperty(object): From 1050766fe1e0f40eaaa55fb0f31dab39667a45a0 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Wed, 13 Nov 2019 15:09:55 -0300 Subject: [PATCH 71/76] Update .Net framework to 4.5.2 - Update .Net framework to 4.5.2 trying to avoid initialization hang - Add a file cache of path directories to avoid having to check if a provided dll exists in each path or not --- src/clrmodule/clrmodule.csproj | 8 +-- src/clrmodule/packages.config | 6 +-- src/console/Console.csproj | 8 +-- src/embed_tests/Python.EmbeddingTest.csproj | 4 +- src/embed_tests/packages.config | 8 +-- src/runtime/Python.Runtime.csproj | 14 +++--- src/runtime/assemblymanager.cs | 55 ++++++++++++++++++--- src/testing/Python.Test.csproj | 8 +-- 8 files changed, 76 insertions(+), 35 deletions(-) diff --git a/src/clrmodule/clrmodule.csproj b/src/clrmodule/clrmodule.csproj index 6e5ff4966..374948d89 100644 --- a/src/clrmodule/clrmodule.csproj +++ b/src/clrmodule/clrmodule.csproj @@ -1,5 +1,5 @@ - - + + Debug AnyCPU @@ -9,7 +9,7 @@ clrmodule bin\clrmodule.xml bin\ - v4.0 + v4.5.2 1591 ..\..\ @@ -92,4 +92,4 @@ - + \ No newline at end of file diff --git a/src/clrmodule/packages.config b/src/clrmodule/packages.config index 2a95dc54d..bfd9f475d 100644 --- a/src/clrmodule/packages.config +++ b/src/clrmodule/packages.config @@ -1,4 +1,4 @@ - + - - + + \ No newline at end of file diff --git a/src/console/Console.csproj b/src/console/Console.csproj index ea88b6356..eb54c9209 100644 --- a/src/console/Console.csproj +++ b/src/console/Console.csproj @@ -1,5 +1,5 @@ - - + + Debug AnyCPU @@ -9,7 +9,7 @@ Python.Runtime bin\nPython.xml bin\ - v4.0 + v4.5.2 1591 ..\..\ @@ -98,4 +98,4 @@ - + \ No newline at end of file diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index b0aa650d3..c2b323979 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -1,5 +1,5 @@  - + Debug AnyCPU @@ -9,7 +9,7 @@ Python.EmbeddingTest bin\Python.EmbeddingTest.xml bin\ - v4.0 + v4.5.2 1591 ..\..\ diff --git a/src/embed_tests/packages.config b/src/embed_tests/packages.config index 8c175f441..b5504a5e4 100644 --- a/src/embed_tests/packages.config +++ b/src/embed_tests/packages.config @@ -1,5 +1,5 @@ - + - - - + + + \ No newline at end of file diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 28ea6424f..1db48d28c 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,5 +1,5 @@  - + Debug AnyCPU @@ -9,7 +9,7 @@ Python.Runtime bin\Python.Runtime.xml bin\ - v4.0 + v4.5.2 1591 ..\..\ @@ -22,11 +22,11 @@ - PYTHON2;PYTHON27;UCS4 diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 1beaf4789..a22db843d 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Linq; using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -26,6 +27,7 @@ internal class AssemblyManager // updated only under GIL? private static Dictionary probed; private static List pypath; + private static Dictionary> filesInPath; private AssemblyManager() { @@ -44,6 +46,7 @@ internal static void Initialize() probed = new Dictionary(32); assemblies = new ConcurrentQueue(); pypath = new List(16); + filesInPath = new Dictionary>(); AppDomain domain = AppDomain.CurrentDomain; @@ -184,6 +187,44 @@ internal static void UpdatePath() pypath.Add(path == string.Empty ? path : path + sep); } } + + // for performance we will search for all files in each directory in the path once + Parallel.ForEach(pypath.Where(s => + { + try + { + lock (filesInPath) + { + // only search in directory if it exists and we haven't already analyzed it + return Directory.Exists(s) && !filesInPath.ContainsKey(s); + } + } + catch + { + // just in case, file operations can throw + } + return false; + }), path => + { + var container = new HashSet(); + try + { + foreach (var file in Directory.EnumerateFiles(path) + .Where(file => file.EndsWith(".dll") || file.EndsWith(".exe"))) + { + container.Add(Path.GetFileName(file)); + } + } + catch + { + // just in case, file operations can throw + } + + lock (filesInPath) + { + filesInPath[path] = container; + } + }); } } @@ -194,17 +235,17 @@ internal static void UpdatePath() /// public static string FindAssembly(string name) { - foreach (var head in pypath) + foreach (var kvp in filesInPath) { - var dll = $"{head}{name}.dll"; - if (File.Exists(dll)) + var dll = $"{name}.dll"; + if (kvp.Value.Contains(dll)) { - return dll; + return kvp.Key + dll; } - var executable = $"{head}{name}.exe"; - if (File.Exists(executable)) + var executable = $"{name}.exe"; + if (kvp.Value.Contains(executable)) { - return executable; + return kvp.Key + executable; } } return null; diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index 8a8d9ed2b..ff9c247fa 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -1,5 +1,5 @@ - - + + Debug AnyCPU @@ -9,7 +9,7 @@ Python.Test bin\Python.Test.xml bin\ - v4.0 + v4.5.2 1591,0067 ..\..\ @@ -110,4 +110,4 @@ - + \ No newline at end of file From 7e916e04fb616c4faf82ba6324bc00f5c0a0af26 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Wed, 13 Nov 2019 15:24:15 -0300 Subject: [PATCH 72/76] Version bump 1.0.5.29 --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index c3f226ae6..f0a2abed2 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.28 +current_version = 1.0.5.29 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index c322f4a39..61f336080 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.28" + version: "1.0.5.29" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 75bfb5c77..512771fa9 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.28", + version="1.0.5.29", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index e1b69d556..a8f3149df 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.28")] +[assembly: AssemblyVersion("1.0.5.29")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 89bbcf1aa..6387df7e5 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.28"), + Version = new Version("1.0.5.29"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 61b127d94..01d0d409b 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.28" +__version__ = "1.0.5.29" class clrproperty(object): From 39774f506e3a5a7d5c94cc83c2d18f8a94e036d8 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Fri, 27 Mar 2020 23:59:32 +0000 Subject: [PATCH 73/76] Version Bump From 1.0.5.29 to 1.0.5.30 --- .bumpversion.cfg | 2 +- conda.recipe/meta.yaml | 2 +- setup.py | 2 +- src/SharedAssemblyInfo.cs | 2 +- src/clrmodule/ClrModule.cs | 2 +- src/runtime/resources/clr.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index f0a2abed2..01b9ae991 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.0.5.29 +current_version = 1.0.5.30 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 61f336080..e2c407435 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "1.0.5.29" + version: "1.0.5.30" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 512771fa9..8e72acc96 100644 --- a/setup.py +++ b/setup.py @@ -485,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="1.0.5.29", + version="1.0.5.30", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index a8f3149df..3d2192f56 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("1.0.5.29")] +[assembly: AssemblyVersion("1.0.5.30")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 6387df7e5..62ec1ad42 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("1.0.5.29"), + Version = new Version("1.0.5.30"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index 01d0d409b..3e35b13ab 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "1.0.5.29" +__version__ = "1.0.5.30" class clrproperty(object): From 755d867cb44e8222b784afe8082e939487efe858 Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Sat, 28 Mar 2020 00:01:53 +0000 Subject: [PATCH 74/76] Implements DictionaryObject Enables `__contains__` and `__len`__ for CLR classes that implements a dictionary. --- src/runtime/Python.Runtime.csproj | 1 + src/runtime/classmanager.cs | 5 + src/runtime/dictionaryobject.cs | 141 ++++++++++++++++++++++++++++ src/testing/Python.Test.csproj | 1 + src/testing/dictionarytest.cs | 106 +++++++++++++++++++++ src/tests/test_dictionary.py | 148 ++++++++++++++++++++++++++++++ 6 files changed, 402 insertions(+) create mode 100644 src/runtime/dictionaryobject.cs create mode 100644 src/testing/dictionarytest.cs create mode 100644 src/tests/test_dictionary.py diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 1db48d28c..d1bdccf07 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -76,6 +76,7 @@ + diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 83495cec4..5aff82d02 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -87,6 +87,11 @@ private static ClassBase CreateClass(Type type) impl = new ArrayObject(type); } + else if (type.IsDictionary()) + { + impl = new DictionaryObject(type); + } + else if (type.IsInterface) { impl = new InterfaceObject(type); diff --git a/src/runtime/dictionaryobject.cs b/src/runtime/dictionaryobject.cs new file mode 100644 index 000000000..8338578c4 --- /dev/null +++ b/src/runtime/dictionaryobject.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections.Generic; +using System.Reflection; + +namespace Python.Runtime +{ + /// + /// Implements a Python type for managed dictionaries. This type is essentially + /// the same as a ClassObject, except that it provides sequence semantics + /// to support natural dictionary usage (__contains__ and __len__) from Python. + /// + internal class DictionaryObject : ClassObject + { + private static Dictionary, MethodInfo> methodsByType = new Dictionary, MethodInfo>(); + private static Dictionary methodMap = new Dictionary + { + { "mp_length", "Count" }, + { "sq_contains", "ContainsKey" } + }; + + public List MappedMethods { get; } = new List(); + + internal DictionaryObject(Type tp) : base(tp) + { + if (!tp.IsDictionary()) + { + throw new ArgumentException("object is not a dict"); + } + + foreach (var name in methodMap) + { + var key = Tuple.Create(type, name.Value); + MethodInfo method; + if (!methodsByType.TryGetValue(key, out method)) + { + method = tp.GetMethod(name.Value); + if (method == null) + { + method = tp.GetMethod($"get_{name.Value}"); + } + if (method == null) + { + continue; + } + methodsByType.Add(key, method); + } + + MappedMethods.Add(name.Key); + } + } + + internal override bool CanSubclass() => false; + + /// + /// Implements __len__ for dictionary types. + /// + public static int mp_length(IntPtr ob) + { + var obj = (CLRObject)GetManagedObject(ob); + var self = obj.inst; + + MethodInfo methodInfo; + if (!TryGetMethodInfo(self.GetType(), "Count", out methodInfo)) + { + return 0; + } + + return (int)methodInfo.Invoke(self, null); + } + + /// + /// Implements __contains__ for dictionary types. + /// + public static int sq_contains(IntPtr ob, IntPtr v) + { + var obj = (CLRObject)GetManagedObject(ob); + var self = obj.inst; + + MethodInfo methodInfo; + if (!TryGetMethodInfo(self.GetType(), "ContainsKey", out methodInfo)) + { + return 0; + } + + var parameters = methodInfo.GetParameters(); + object arg; + if (!Converter.ToManaged(v, parameters[0].ParameterType, out arg, false)) + { + Exceptions.SetError(Exceptions.TypeError, + $"invalid parameter type for sq_contains: should be {Converter.GetTypeByAlias(v)}, found {parameters[0].ParameterType}"); + } + + return (bool)methodInfo.Invoke(self, new[] { arg }) ? 1 : 0; + } + + private static bool TryGetMethodInfo(Type type, string alias, out MethodInfo methodInfo) + { + var key = Tuple.Create(type, alias); + + if (!methodsByType.TryGetValue(key, out methodInfo)) + { + Exceptions.SetError(Exceptions.TypeError, + $"{nameof(type)} does not define {alias} method"); + + return false; + } + + return true; + } + } + + public static class DictionaryObjectExtension + { + public static bool IsDictionary(this Type type) + { + var iEnumerableType = typeof(IEnumerable<>); + var keyValuePairType = typeof(KeyValuePair<,>); + + var interfaces = type.GetInterfaces(); + foreach (var i in interfaces) + { + if (i.IsGenericType && + i.GetGenericTypeDefinition() == iEnumerableType) + { + var arguments = i.GetGenericArguments(); + if (arguments.Length != 1) continue; + + var a = arguments[0]; + if (a.IsGenericType && + a.GetGenericTypeDefinition() == keyValuePairType && + a.GetGenericArguments().Length == 2) + { + return true; + } + } + } + + return false; + } + } +} diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index ff9c247fa..434da99e1 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -70,6 +70,7 @@ pdbonly + diff --git a/src/testing/dictionarytest.cs b/src/testing/dictionarytest.cs new file mode 100644 index 000000000..a7fa3497d --- /dev/null +++ b/src/testing/dictionarytest.cs @@ -0,0 +1,106 @@ +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +namespace Python.Test +{ + /// + /// Supports units tests for dictionary __contains__ and __len__ + /// + public class PublicDictionaryTest + { + public IDictionary items; + + public PublicDictionaryTest() + { + items = new int[5] { 0, 1, 2, 3, 4 } + .ToDictionary(k => k.ToString(), v => v); + } + } + + + public class ProtectedDictionaryTest + { + protected IDictionary items; + + public ProtectedDictionaryTest() + { + items = new int[5] { 0, 1, 2, 3, 4 } + .ToDictionary(k => k.ToString(), v => v); + } + } + + + public class InternalDictionaryTest + { + internal IDictionary items; + + public InternalDictionaryTest() + { + items = new int[5] { 0, 1, 2, 3, 4 } + .ToDictionary(k => k.ToString(), v => v); + } + } + + + public class PrivateDictionaryTest + { + private IDictionary items; + + public PrivateDictionaryTest() + { + items = new int[5] { 0, 1, 2, 3, 4 } + .ToDictionary(k => k.ToString(), v => v); + } + } + + public class InheritedDictionaryTest : IDictionary + { + private readonly IDictionary items; + + public InheritedDictionaryTest() + { + items = new int[5] { 0, 1, 2, 3, 4 } + .ToDictionary(k => k.ToString(), v => v); + } + + public int this[string key] + { + get { return items[key]; } + set { items[key] = value; } + } + + public ICollection Keys => items.Keys; + + public ICollection Values => items.Values; + + public int Count => items.Count; + + public bool IsReadOnly => false; + + public void Add(string key, int value) => items.Add(key, value); + + public void Add(KeyValuePair item) => items.Add(item); + + public void Clear() => items.Clear(); + + public bool Contains(KeyValuePair item) => items.Contains(item); + + public bool ContainsKey(string key) => items.ContainsKey(key); + + public void CopyTo(KeyValuePair[] array, int arrayIndex) + { + items.CopyTo(array, arrayIndex); + } + + public IEnumerator> GetEnumerator() => items.GetEnumerator(); + + public bool Remove(string key) => items.Remove(key); + + public bool Remove(KeyValuePair item) => items.Remove(item); + + public bool TryGetValue(string key, out int value) => items.TryGetValue(key, out value); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } +} diff --git a/src/tests/test_dictionary.py b/src/tests/test_dictionary.py new file mode 100644 index 000000000..98c3cfb20 --- /dev/null +++ b/src/tests/test_dictionary.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- + +"""Test support for managed dictionaries.""" + +import Python.Test as Test +import System +import pytest + +from ._compat import PY2, UserList, long, range, unichr + + +def test_public_dict(): + """Test public dict.""" + ob = Test.PublicDictionaryTest() + items = ob.items + + assert len(items) == 5 + + assert items['0'] == 0 + assert items['4'] == 4 + + items['0'] = 8 + assert items['0'] == 8 + + items['4'] = 9 + assert items['4'] == 9 + + items['-4'] = 0 + assert items['-4'] == 0 + + items['-1'] = 4 + assert items['-1'] == 4 + +def test_protected_dict(): + """Test protected dict.""" + ob = Test.ProtectedDictionaryTest() + items = ob.items + + assert len(items) == 5 + + assert items['0'] == 0 + assert items['4'] == 4 + + items['0'] = 8 + assert items['0'] == 8 + + items['4'] = 9 + assert items['4'] == 9 + + items['-4'] = 0 + assert items['-4'] == 0 + + items['-1'] = 4 + assert items['-1'] == 4 + +def test_internal_dict(): + """Test internal dict.""" + + with pytest.raises(AttributeError): + ob = Test.InternalDictionaryTest() + _ = ob.items + +def test_private_dict(): + """Test private dict.""" + + with pytest.raises(AttributeError): + ob = Test.PrivateDictionaryTest() + _ = ob.items + +def test_dict_contains(): + """Test dict support for __contains__.""" + + ob = Test.PublicDictionaryTest() + items = ob.items + + assert '0' in items + assert '1' in items + assert '2' in items + assert '3' in items + assert '4' in items + + assert not ('5' in items) + assert not ('-1' in items) + +def test_dict_abuse(): + """Test dict abuse.""" + _class = Test.PublicDictionaryTest + ob = Test.PublicDictionaryTest() + + with pytest.raises(AttributeError): + del _class.__getitem__ + + with pytest.raises(AttributeError): + del ob.__getitem__ + + with pytest.raises(AttributeError): + del _class.__setitem__ + + with pytest.raises(AttributeError): + del ob.__setitem__ + + with pytest.raises(TypeError): + Test.PublicArrayTest.__getitem__(0, 0) + + with pytest.raises(TypeError): + Test.PublicArrayTest.__setitem__(0, 0, 0) + + with pytest.raises(TypeError): + desc = Test.PublicArrayTest.__dict__['__getitem__'] + desc(0, 0) + + with pytest.raises(TypeError): + desc = Test.PublicArrayTest.__dict__['__setitem__'] + desc(0, 0, 0) + +def test_InheritedDictionary(): + """Test class that inherited from IDictionary.""" + items = Test.InheritedDictionaryTest() + + assert len(items) == 5 + + assert items['0'] == 0 + assert items['4'] == 4 + + items['0'] = 8 + assert items['0'] == 8 + + items['4'] = 9 + assert items['4'] == 9 + + items['-4'] = 0 + assert items['-4'] == 0 + + items['-1'] = 4 + assert items['-1'] == 4 + +def test_InheritedDictionary_contains(): + """Test dict support for __contains__ in class that inherited from IDictionary""" + items = Test.InheritedDictionaryTest() + + assert '0' in items + assert '1' in items + assert '2' in items + assert '3' in items + assert '4' in items + + assert not ('5' in items) + assert not ('-1' in items) \ No newline at end of file From 074d01e132fd2a4c3651e5e986bd15eca46234fb Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Tue, 31 Mar 2020 00:54:40 +0100 Subject: [PATCH 75/76] Changes DictionaryObject to KeyValuePairEnumerableObject It is more generical. Verifies the existence of `Count` and `ContainsKey` that are necessary for `__len__` and `__contains__`. --- src/runtime/Python.Runtime.csproj | 2 +- src/runtime/classmanager.cs | 4 +- ...ect.cs => keyvaluepairenumerableobject.cs} | 62 +++++++++---------- 3 files changed, 31 insertions(+), 37 deletions(-) rename src/runtime/{dictionaryobject.cs => keyvaluepairenumerableobject.cs} (68%) diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index d1bdccf07..3ef2ad471 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -76,7 +76,7 @@ - + diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 5aff82d02..837cdd16e 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -87,9 +87,9 @@ private static ClassBase CreateClass(Type type) impl = new ArrayObject(type); } - else if (type.IsDictionary()) + else if (type.IsKeyValuePairEnumerable()) { - impl = new DictionaryObject(type); + impl = new KeyValuePairEnumerableObject(type); } else if (type.IsInterface) diff --git a/src/runtime/dictionaryobject.cs b/src/runtime/keyvaluepairenumerableobject.cs similarity index 68% rename from src/runtime/dictionaryobject.cs rename to src/runtime/keyvaluepairenumerableobject.cs index 8338578c4..3f7afc852 100644 --- a/src/runtime/dictionaryobject.cs +++ b/src/runtime/keyvaluepairenumerableobject.cs @@ -5,11 +5,12 @@ namespace Python.Runtime { /// - /// Implements a Python type for managed dictionaries. This type is essentially - /// the same as a ClassObject, except that it provides sequence semantics - /// to support natural dictionary usage (__contains__ and __len__) from Python. + /// Implements a Python type for managed KeyValuePairEnumerable (dictionaries). + /// This type is essentially the same as a ClassObject, except that it provides + /// sequence semantics to support natural dictionary usage (__contains__ and __len__) + /// from Python. /// - internal class DictionaryObject : ClassObject + internal class KeyValuePairEnumerableObject : ClassObject { private static Dictionary, MethodInfo> methodsByType = new Dictionary, MethodInfo>(); private static Dictionary methodMap = new Dictionary @@ -20,11 +21,11 @@ internal class DictionaryObject : ClassObject public List MappedMethods { get; } = new List(); - internal DictionaryObject(Type tp) : base(tp) + internal KeyValuePairEnumerableObject(Type tp) : base(tp) { - if (!tp.IsDictionary()) + if (!tp.IsKeyValuePairEnumerable()) { - throw new ArgumentException("object is not a dict"); + throw new ArgumentException("object is not a KeyValuePair Enumerable"); } foreach (var name in methodMap) @@ -59,11 +60,8 @@ public static int mp_length(IntPtr ob) var obj = (CLRObject)GetManagedObject(ob); var self = obj.inst; - MethodInfo methodInfo; - if (!TryGetMethodInfo(self.GetType(), "Count", out methodInfo)) - { - return 0; - } + var key = Tuple.Create(self.GetType(), "Count"); + var methodInfo = methodsByType[key]; return (int)methodInfo.Invoke(self, null); } @@ -76,11 +74,8 @@ public static int sq_contains(IntPtr ob, IntPtr v) var obj = (CLRObject)GetManagedObject(ob); var self = obj.inst; - MethodInfo methodInfo; - if (!TryGetMethodInfo(self.GetType(), "ContainsKey", out methodInfo)) - { - return 0; - } + var key = Tuple.Create(self.GetType(), "ContainsKey"); + var methodInfo = methodsByType[key]; var parameters = methodInfo.GetParameters(); object arg; @@ -92,29 +87,15 @@ public static int sq_contains(IntPtr ob, IntPtr v) return (bool)methodInfo.Invoke(self, new[] { arg }) ? 1 : 0; } - - private static bool TryGetMethodInfo(Type type, string alias, out MethodInfo methodInfo) - { - var key = Tuple.Create(type, alias); - - if (!methodsByType.TryGetValue(key, out methodInfo)) - { - Exceptions.SetError(Exceptions.TypeError, - $"{nameof(type)} does not define {alias} method"); - - return false; - } - - return true; - } } - public static class DictionaryObjectExtension + public static class KeyValuePairEnumerableObjectExtension { - public static bool IsDictionary(this Type type) + public static bool IsKeyValuePairEnumerable(this Type type) { var iEnumerableType = typeof(IEnumerable<>); var keyValuePairType = typeof(KeyValuePair<,>); + var requiredMethods = new[] { "ContainsKey", "Count" }; var interfaces = type.GetInterfaces(); foreach (var i in interfaces) @@ -130,6 +111,19 @@ public static bool IsDictionary(this Type type) a.GetGenericTypeDefinition() == keyValuePairType && a.GetGenericArguments().Length == 2) { + foreach (var requiredMethod in requiredMethods) + { + var method = type.GetMethod(requiredMethod); + if (method == null) + { + method = type.GetMethod($"get_{requiredMethod}"); + if (method == null) + { + return false; + } + } + } + return true; } } From 49245d6f13514cfab8655875929000426eb8104a Mon Sep 17 00:00:00 2001 From: AlexCatarino Date: Tue, 31 Mar 2020 02:00:44 +0100 Subject: [PATCH 76/76] Addresses Peer-Review: Perfomance Improments --- src/runtime/keyvaluepairenumerableobject.cs | 57 ++++++--------------- 1 file changed, 17 insertions(+), 40 deletions(-) diff --git a/src/runtime/keyvaluepairenumerableobject.cs b/src/runtime/keyvaluepairenumerableobject.cs index 3f7afc852..c1644442c 100644 --- a/src/runtime/keyvaluepairenumerableobject.cs +++ b/src/runtime/keyvaluepairenumerableobject.cs @@ -13,41 +13,32 @@ namespace Python.Runtime internal class KeyValuePairEnumerableObject : ClassObject { private static Dictionary, MethodInfo> methodsByType = new Dictionary, MethodInfo>(); - private static Dictionary methodMap = new Dictionary - { - { "mp_length", "Count" }, - { "sq_contains", "ContainsKey" } - }; - - public List MappedMethods { get; } = new List(); + private static List requiredMethods = new List { "Count", "ContainsKey" }; - internal KeyValuePairEnumerableObject(Type tp) : base(tp) + internal static bool VerifyMethodRequirements(Type type) { - if (!tp.IsKeyValuePairEnumerable()) - { - throw new ArgumentException("object is not a KeyValuePair Enumerable"); - } - - foreach (var name in methodMap) + foreach (var requiredMethod in requiredMethods) { - var key = Tuple.Create(type, name.Value); - MethodInfo method; - if (!methodsByType.TryGetValue(key, out method)) + var method = type.GetMethod(requiredMethod); + if (method == null) { - method = tp.GetMethod(name.Value); + method = type.GetMethod($"get_{requiredMethod}"); if (method == null) { - method = tp.GetMethod($"get_{name.Value}"); + return false; } - if (method == null) - { - continue; - } - methodsByType.Add(key, method); } - MappedMethods.Add(name.Key); + var key = Tuple.Create(type, requiredMethod); + methodsByType.Add(key, method); } + + return true; + } + + internal KeyValuePairEnumerableObject(Type tp) : base(tp) + { + } internal override bool CanSubclass() => false; @@ -95,7 +86,6 @@ public static bool IsKeyValuePairEnumerable(this Type type) { var iEnumerableType = typeof(IEnumerable<>); var keyValuePairType = typeof(KeyValuePair<,>); - var requiredMethods = new[] { "ContainsKey", "Count" }; var interfaces = type.GetInterfaces(); foreach (var i in interfaces) @@ -111,20 +101,7 @@ public static bool IsKeyValuePairEnumerable(this Type type) a.GetGenericTypeDefinition() == keyValuePairType && a.GetGenericArguments().Length == 2) { - foreach (var requiredMethod in requiredMethods) - { - var method = type.GetMethod(requiredMethod); - if (method == null) - { - method = type.GetMethod($"get_{requiredMethod}"); - if (method == null) - { - return false; - } - } - } - - return true; + return KeyValuePairEnumerableObject.VerifyMethodRequirements(type); } } }