diff --git a/AUTHORS.md b/AUTHORS.md index 69e7b5c4a..167fd496c 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -24,6 +24,7 @@ - BenoƮt Hudson ([@benoithudson](https://github.com/benoithudson)) - Bradley Friedman ([@leith-bartrich](https://github.com/leith-bartrich)) - Callum Noble ([@callumnoble](https://github.com/callumnoble)) +- Christabella Irwanto([@christabella](https://github.com/christabella)) - Christian Heimes ([@tiran](https://github.com/tiran)) - Christoph Gohlke ([@cgohlke](https://github.com/cgohlke)) - Christopher Bremner ([@chrisjbremner](https://github.com/chrisjbremner)) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53c45f419..a9a804e8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added - Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax +- Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). ### Changed - Drop support for Python 2, 3.4, and 3.5 diff --git a/src/embed_tests/TestOperator.cs b/src/embed_tests/TestOperator.cs new file mode 100644 index 000000000..ecdb0c1dc --- /dev/null +++ b/src/embed_tests/TestOperator.cs @@ -0,0 +1,332 @@ +using NUnit.Framework; + +using Python.Runtime; + +using System.Linq; +using System.Reflection; + +namespace Python.EmbeddingTest +{ + public class TestOperator + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + public class OperableObject + { + public int Num { get; set; } + + public OperableObject(int num) + { + Num = num; + } + + public static OperableObject operator ~(OperableObject a) + { + return new OperableObject(~a.Num); + } + + public static OperableObject operator +(OperableObject a) + { + return new OperableObject(+a.Num); + } + + public static OperableObject operator -(OperableObject a) + { + return new OperableObject(-a.Num); + } + + public static OperableObject operator +(int a, OperableObject b) + { + return new OperableObject(a + b.Num); + } + public static OperableObject operator +(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num + b.Num); + } + public static OperableObject operator +(OperableObject a, int b) + { + return new OperableObject(a.Num + b); + } + + public static OperableObject operator -(int a, OperableObject b) + { + return new OperableObject(a - b.Num); + } + public static OperableObject operator -(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num - b.Num); + } + public static OperableObject operator -(OperableObject a, int b) + { + return new OperableObject(a.Num - b); + } + + public static OperableObject operator *(int a, OperableObject b) + { + return new OperableObject(a * b.Num); + } + public static OperableObject operator *(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num * b.Num); + } + public static OperableObject operator *(OperableObject a, int b) + { + return new OperableObject(a.Num * b); + } + + public static OperableObject operator /(int a, OperableObject b) + { + return new OperableObject(a / b.Num); + } + public static OperableObject operator /(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num / b.Num); + } + public static OperableObject operator /(OperableObject a, int b) + { + return new OperableObject(a.Num / b); + } + + public static OperableObject operator %(int a, OperableObject b) + { + return new OperableObject(a % b.Num); + } + public static OperableObject operator %(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num % b.Num); + } + public static OperableObject operator %(OperableObject a, int b) + { + return new OperableObject(a.Num % b); + } + + public static OperableObject operator &(int a, OperableObject b) + { + return new OperableObject(a & b.Num); + } + public static OperableObject operator &(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num & b.Num); + } + public static OperableObject operator &(OperableObject a, int b) + { + return new OperableObject(a.Num & b); + } + + public static OperableObject operator |(int a, OperableObject b) + { + return new OperableObject(a | b.Num); + } + public static OperableObject operator |(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num | b.Num); + } + public static OperableObject operator |(OperableObject a, int b) + { + return new OperableObject(a.Num | b); + } + + public static OperableObject operator ^(int a, OperableObject b) + { + return new OperableObject(a ^ b.Num); + } + public static OperableObject operator ^(OperableObject a, OperableObject b) + { + return new OperableObject(a.Num ^ b.Num); + } + public static OperableObject operator ^(OperableObject a, int b) + { + return new OperableObject(a.Num ^ b); + } + + public static OperableObject operator <<(OperableObject a, int offset) + { + return new OperableObject(a.Num << offset); + } + + public static OperableObject operator >>(OperableObject a, int offset) + { + return new OperableObject(a.Num >> offset); + } + } + + [Test] + public void OperatorOverloads() + { + string name = string.Format("{0}.{1}", + typeof(OperableObject).DeclaringType.Name, + typeof(OperableObject).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = cls(-2) +b = cls(10) +c = ~a +assert c.Num == ~a.Num + +c = +a +assert c.Num == +a.Num + +a = cls(2) +c = -a +assert c.Num == -a.Num + +c = a + b +assert c.Num == a.Num + b.Num + +c = a - b +assert c.Num == a.Num - b.Num + +c = a * b +assert c.Num == a.Num * b.Num + +c = a / b +assert c.Num == a.Num // b.Num + +c = a % b +assert c.Num == a.Num % b.Num + +c = a & b +assert c.Num == a.Num & b.Num + +c = a | b +assert c.Num == a.Num | b.Num + +c = a ^ b +assert c.Num == a.Num ^ b.Num +"); + } + + [Test] + public void OperatorOverloadMissingArgument() + { + string name = string.Format("{0}.{1}", + typeof(OperableObject).DeclaringType.Name, + typeof(OperableObject).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + Assert.Throws(() => + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = cls(2) +b = cls(10) +a.op_Addition() +")); + } + + [Test] + public void ForwardOperatorOverloads() + { + string name = string.Format("{0}.{1}", + typeof(OperableObject).DeclaringType.Name, + typeof(OperableObject).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = cls(2) +b = 10 +c = a + b +assert c.Num == a.Num + b + +c = a - b +assert c.Num == a.Num - b + +c = a * b +assert c.Num == a.Num * b + +c = a / b +assert c.Num == a.Num // b + +c = a % b +assert c.Num == a.Num % b + +c = a & b +assert c.Num == a.Num & b + +c = a | b +assert c.Num == a.Num | b + +c = a ^ b +assert c.Num == a.Num ^ b +"); + } + + + [Test] + public void ReverseOperatorOverloads() + { + string name = string.Format("{0}.{1}", + typeof(OperableObject).DeclaringType.Name, + typeof(OperableObject).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = 2 +b = cls(10) + +c = a + b +assert c.Num == a + b.Num + +c = a - b +assert c.Num == a - b.Num + +c = a * b +assert c.Num == a * b.Num + +c = a / b +assert c.Num == a // b.Num + +c = a % b +assert c.Num == a % b.Num + +c = a & b +assert c.Num == a & b.Num + +c = a | b +assert c.Num == a | b.Num + +c = a ^ b +assert c.Num == a ^ b.Num +"); + + } + [Test] + public void ShiftOperatorOverloads() + { + string name = string.Format("{0}.{1}", + typeof(OperableObject).DeclaringType.Name, + typeof(OperableObject).Name); + string module = MethodBase.GetCurrentMethod().DeclaringType.Namespace; + + PythonEngine.Exec($@" +from {module} import * +cls = {name} +a = cls(2) +b = cls(10) + +c = a << b.Num +assert c.Num == a.Num << b.Num + +c = a >> b.Num +assert c.Num == a.Num >> b.Num +"); + } + } +} diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index c8bed6bc4..db4146722 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -470,6 +470,19 @@ private static ClassInfo GetClassInfo(Type type) ob = new MethodObject(type, name, mlist); ci.members[name] = ob; + if (mlist.Any(OperatorMethod.IsOperatorMethod)) + { + string pyName = OperatorMethod.GetPyMethodName(name); + string pyNameReverse = OperatorMethod.ReversePyMethodName(pyName); + MethodInfo[] forwardMethods, reverseMethods; + OperatorMethod.FilterMethods(mlist, out forwardMethods, out reverseMethods); + // Only methods where the left operand is the declaring type. + if (forwardMethods.Length > 0) + ci.members[pyName] = new MethodObject(type, name, forwardMethods); + // Only methods where only the right operand is the declaring type. + if (reverseMethods.Length > 0) + ci.members[pyNameReverse] = new MethodObject(type, name, reverseMethods); + } } if (ci.indexer == null && type.IsClass) diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 2cf548f48..47883f0e6 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -342,20 +342,59 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth bool paramsArray; int kwargsMatched; int defaultsNeeded; - - if (!MatchesArgumentCount(pynargs, pi, kwargDict, out paramsArray, out defaultArgList, out kwargsMatched, out defaultsNeeded)) + bool isOperator = OperatorMethod.IsOperatorMethod(mi); + int clrnargs = pi.Length; + // Binary operator methods will have 2 CLR args but only one Python arg + // (unary operators will have 1 less each), since Python operator methods are bound. + isOperator = isOperator && pynargs == clrnargs - 1; + if (!MatchesArgumentCount(pynargs, pi, kwargDict, out paramsArray, out defaultArgList, out kwargsMatched, out defaultsNeeded) && !isOperator) { continue; } + // Preprocessing pi to remove either the first or second argument. + bool isReverse = isOperator && OperatorMethod.IsReverse((MethodInfo)mi); // Only cast if isOperator. + if (isOperator && !isReverse) { + // The first Python arg is the right operand, while the bound instance is the left. + // We need to skip the first (left operand) CLR argument. + pi = pi.Skip(1).ToArray(); + } + else if (isOperator && isReverse) { + // The first Python arg is the left operand. + // We need to take the first CLR argument. + pi = pi.Take(1).ToArray(); + } var outs = 0; var margs = TryConvertArguments(pi, paramsArray, args, pynargs, kwargDict, defaultArgList, - needsResolution: _methods.Length > 1, + needsResolution: _methods.Length > 1, // If there's more than one possible match. outs: out outs); - if (margs == null) { continue; } + if (isOperator) + { + if (inst != IntPtr.Zero) + { + if (ManagedType.GetManagedObject(inst) is CLRObject co) + { + bool isUnary = pynargs == 0; + // Postprocessing to extend margs. + var margsTemp = isUnary ? new object[1] : new object[2]; + // If reverse, the bound instance is the right operand. + int boundOperandIndex = isReverse ? 1 : 0; + // If reverse, the passed instance is the left operand. + int passedOperandIndex = isReverse ? 0 : 1; + margsTemp[boundOperandIndex] = co.inst; + if (!isUnary) + { + margsTemp[passedOperandIndex] = margs[0]; + } + margs = margsTemp; + } + else { break; } + } + } + var matchedMethod = new MatchedMethod(kwargsMatched, defaultsNeeded, margs, outs, mi); argMatchedMethods.Add(matchedMethod); @@ -543,6 +582,15 @@ static object[] TryConvertArguments(ParameterInfo[] pi, bool paramsArray, return margs; } + /// + /// Try to convert a Python argument object to a managed CLR type. + /// + /// Pointer to the object at a particular parameter. + /// That parameter's managed type. + /// There are multiple overloading methods that need resolution. + /// Converted argument. + /// Whether the CLR type is passed by reference. + /// static bool TryConvertArgument(IntPtr op, Type parameterType, bool needsResolution, out object arg, out bool isOut) { @@ -633,7 +681,17 @@ static Type TryComputeClrArgumentType(Type parameterType, IntPtr argument, bool return clrtype; } - + /// + /// Check whether the number of Python and .NET arguments match, and compute additional arg information. + /// + /// Number of positional args passed from Python. + /// Parameters of the specified .NET method. + /// Keyword args passed from Python. + /// True if the final param of the .NET method is an array (`params` keyword). + /// List of default values for arguments. + /// Number of kwargs from Python that are also present in the .NET method. + /// Number of non-null defaultsArgs. + /// static bool MatchesArgumentCount(int positionalArgumentCount, ParameterInfo[] parameters, Dictionary kwargDict, out bool paramsArray, @@ -644,19 +702,18 @@ static bool MatchesArgumentCount(int positionalArgumentCount, ParameterInfo[] pa defaultArgList = null; var match = false; paramsArray = parameters.Length > 0 ? Attribute.IsDefined(parameters[parameters.Length - 1], typeof(ParamArrayAttribute)) : false; - var kwargCount = kwargDict.Count; kwargsMatched = 0; defaultsNeeded = 0; - if (positionalArgumentCount == parameters.Length && kwargDict.Count == 0) { match = true; } else if (positionalArgumentCount < parameters.Length && (!paramsArray || positionalArgumentCount == parameters.Length - 1)) { - // every parameter past 'positionalArgumentCount' must have either - // a corresponding keyword argument or a default parameter match = true; + // every parameter past 'positionalArgumentCount' must have either + // a corresponding keyword arg or a default param, unless the method + // method accepts a params array (which cannot have a default value) defaultArgList = new ArrayList(); for (var v = positionalArgumentCount; v < parameters.Length; v++) { diff --git a/src/runtime/native/ITypeOffsets.cs b/src/runtime/native/ITypeOffsets.cs index 31344c66d..485c041f8 100644 --- a/src/runtime/native/ITypeOffsets.cs +++ b/src/runtime/native/ITypeOffsets.cs @@ -14,7 +14,19 @@ interface ITypeOffsets int mp_length { get; } int mp_subscript { get; } int name { get; } + int nb_positive { get; } + int nb_negative { get; } int nb_add { get; } + int nb_subtract { get; } + int nb_multiply { get; } + int nb_true_divide { get; } + int nb_and { get; } + int nb_or { get; } + int nb_xor { get; } + int nb_lshift { get; } + int nb_rshift { get; } + int nb_remainder { get; } + int nb_invert { get; } int nb_inplace_add { get; } int nb_inplace_subtract { get; } int ob_size { get; } diff --git a/src/runtime/native/TypeOffset.cs b/src/runtime/native/TypeOffset.cs index bca191565..4c1bcefa0 100644 --- a/src/runtime/native/TypeOffset.cs +++ b/src/runtime/native/TypeOffset.cs @@ -21,7 +21,19 @@ static partial class TypeOffset internal static int mp_length { get; private set; } internal static int mp_subscript { get; private set; } internal static int name { get; private set; } + internal static int nb_positive { get; private set; } + internal static int nb_negative { get; private set; } internal static int nb_add { get; private set; } + internal static int nb_subtract { get; private set; } + internal static int nb_multiply { get; private set; } + internal static int nb_true_divide { get; private set; } + internal static int nb_and { get; private set; } + internal static int nb_or { get; private set; } + internal static int nb_xor { get; private set; } + internal static int nb_lshift { get; private set; } + internal static int nb_rshift { get; private set; } + internal static int nb_remainder { get; private set; } + internal static int nb_invert { get; private set; } internal static int nb_inplace_add { get; private set; } internal static int nb_inplace_subtract { get; private set; } internal static int ob_size { get; private set; } diff --git a/src/runtime/operatormethod.cs b/src/runtime/operatormethod.cs new file mode 100644 index 000000000..1e0244510 --- /dev/null +++ b/src/runtime/operatormethod.cs @@ -0,0 +1,172 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Reflection; +using System.Runtime.InteropServices; +using System.Text; + +namespace Python.Runtime +{ + internal static class OperatorMethod + { + /// + /// Maps the compiled method name in .NET CIL (e.g. op_Addition) to + /// the equivalent Python operator (e.g. __add__) as well as the offset + /// that identifies that operator's slot (e.g. nb_add) in heap space. + /// + public static Dictionary OpMethodMap { get; private set; } + public readonly struct SlotDefinition + { + public SlotDefinition(string methodName, int typeOffset) + { + MethodName = methodName; + TypeOffset = typeOffset; + } + public string MethodName { get; } + public int TypeOffset { get; } + } + private static PyObject _opType; + + static OperatorMethod() + { + // .NET operator method names are documented at: + // https://docs.microsoft.com/en-us/dotnet/standard/design-guidelines/operator-overloads + // Python operator methods and slots are documented at: + // https://docs.python.org/3/c-api/typeobj.html + // TODO: Rich compare, inplace operator support + OpMethodMap = new Dictionary + { + ["op_Addition"] = new SlotDefinition("__add__", TypeOffset.nb_add), + ["op_Subtraction"] = new SlotDefinition("__sub__", TypeOffset.nb_subtract), + ["op_Multiply"] = new SlotDefinition("__mul__", TypeOffset.nb_multiply), + ["op_Division"] = new SlotDefinition("__truediv__", TypeOffset.nb_true_divide), + ["op_Modulus"] = new SlotDefinition("__mod__", TypeOffset.nb_remainder), + ["op_BitwiseAnd"] = new SlotDefinition("__and__", TypeOffset.nb_and), + ["op_BitwiseOr"] = new SlotDefinition("__or__", TypeOffset.nb_or), + ["op_ExclusiveOr"] = new SlotDefinition("__xor__", TypeOffset.nb_xor), + ["op_LeftShift"] = new SlotDefinition("__lshift__", TypeOffset.nb_lshift), + ["op_RightShift"] = new SlotDefinition("__rshift__", TypeOffset.nb_rshift), + ["op_OnesComplement"] = new SlotDefinition("__invert__", TypeOffset.nb_invert), + ["op_UnaryNegation"] = new SlotDefinition("__neg__", TypeOffset.nb_negative), + ["op_UnaryPlus"] = new SlotDefinition("__pos__", TypeOffset.nb_positive), + }; + } + + public static void Initialize() + { + _opType = GetOperatorType(); + } + + public static void Shutdown() + { + if (_opType != null) + { + _opType.Dispose(); + _opType = null; + } + } + + public static bool IsOperatorMethod(MethodBase method) + { + if (!method.IsSpecialName) + { + return false; + } + return OpMethodMap.ContainsKey(method.Name); + } + /// + /// For the operator methods of a CLR type, set the special slots of the + /// corresponding Python type's operator methods. + /// + /// + /// + public static void FixupSlots(IntPtr pyType, Type clrType) + { + const BindingFlags flags = BindingFlags.Public | BindingFlags.Static; + Debug.Assert(_opType != null); + foreach (var method in clrType.GetMethods(flags)) + { + if (!IsOperatorMethod(method)) + { + continue; + } + int offset = OpMethodMap[method.Name].TypeOffset; + // Copy the default implementation of e.g. the nb_add slot, + // which simply calls __add__ on the type. + IntPtr func = Marshal.ReadIntPtr(_opType.Handle, offset); + // Write the slot definition of the target Python type, so + // that we can later modify __add___ and it will be called + // when used with a Python operator. + // https://tenthousandmeters.com/blog/python-behind-the-scenes-6-how-python-object-system-works/ + Marshal.WriteIntPtr(pyType, offset, func); + + } + } + + public static string GetPyMethodName(string clrName) + { + return OpMethodMap[clrName].MethodName; + } + + private static string GenerateDummyCode() + { + StringBuilder sb = new StringBuilder(); + sb.AppendLine("class OperatorMethod(object):"); + foreach (var item in OpMethodMap.Values) + { + string def = string.Format(" def {0}(self, other): pass", item.MethodName); + sb.AppendLine(def); + } + return sb.ToString(); + } + + private static PyObject GetOperatorType() + { + using (PyDict locals = new PyDict()) + { + // A hack way for getting typeobject.c::slotdefs + string code = GenerateDummyCode(); + // The resulting OperatorMethod class is stored in a PyDict. + PythonEngine.Exec(code, null, locals.Handle); + // Return the class itself, which is a type. + return locals.GetItem("OperatorMethod"); + } + } + + public static string ReversePyMethodName(string pyName) + { + return pyName.Insert(2, "r"); + } + + /// + /// Check if the method is performing a reverse operation. + /// + /// The operator method. + /// + public static bool IsReverse(MethodInfo method) + { + Type declaringType = method.DeclaringType; + Type leftOperandType = method.GetParameters()[0].ParameterType; + return leftOperandType != declaringType; + } + + public static void FilterMethods(MethodInfo[] methods, out MethodInfo[] forwardMethods, out MethodInfo[] reverseMethods) + { + List forwardMethodsList = new List(); + List reverseMethodsList = new List(); + foreach (var method in methods) + { + if (IsReverse(method)) + { + reverseMethodsList.Add(method); + } else + { + forwardMethodsList.Add(method); + } + + } + forwardMethods = forwardMethodsList.ToArray(); + reverseMethods = reverseMethodsList.ToArray(); + } + } +} diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index f80db04b6..1e8db8278 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -160,6 +160,7 @@ internal static void Initialize(bool initSigs = false, ShutdownMode mode = Shutd // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); + OperatorMethod.Initialize(); if (mode == ShutdownMode.Reload && RuntimeData.HasStashData()) { RuntimeData.RestoreRuntimeData(); @@ -345,6 +346,7 @@ internal static void Shutdown(ShutdownMode mode) RuntimeData.Stash(); } AssemblyManager.Shutdown(); + OperatorMethod.Shutdown(); ImportHook.Shutdown(); ClearClrModules(); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index 49a46cb72..31682c519 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -273,6 +273,7 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) | TypeFlags.HaveGC; Util.WriteCLong(type, TypeOffset.tp_flags, flags); + OperatorMethod.FixupSlots(type, clrType); // Leverage followup initialization from the Python runtime. Note // that the type of the new type must PyType_Type at the time we // call this, else PyType_Ready will skip some slot initialization.