pythonnet · lostmsu · Jan 7, 2022 · Jan 6, 2022
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,6 +13,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][].
 -   Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]).
 -   Add GetPythonThreadID and Interrupt methods in PythonEngine
 -   Ability to implement delegates with `ref` and `out` parameters in Python, by returning the modified parameter values in a tuple. ([#1355][i1355])
+-   Ability to override .NET methods that have `out` or `ref` in Python by returning the modified parameter values in a tuple. ([#1481][i1481])
 -   `PyType` - a wrapper for Python type objects, that also permits creating new heap types from `TypeSpec`
 -    Improved exception handling:
   *   exceptions can now be converted with codecs
@@ -879,3 +880,4 @@ This version improves performance on benchmarks significantly compared to 2.3.
 [i449]: https://github.com/pythonnet/pythonnet/issues/449
 [i1342]: https://github.com/pythonnet/pythonnet/issues/1342
 [i238]: https://github.com/pythonnet/pythonnet/issues/238
+[i1481]: https://github.com/pythonnet/pythonnet/issues/1481
diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs
@@ -429,24 +429,33 @@ private static void AddVirtualMethod(MethodInfo method, Type baseType, TypeBuild
                 il.Emit(OpCodes.Ldloc_0);
                 il.Emit(OpCodes.Ldc_I4, i);
                 il.Emit(OpCodes.Ldarg, i + 1);
-                if (parameterTypes[i].IsValueType)
+                var type = parameterTypes[i];
+                if (type.IsByRef)
                 {
-                    il.Emit(OpCodes.Box, parameterTypes[i]);
+                    type = type.GetElementType();
+                    il.Emit(OpCodes.Ldobj, type);
+                }
+                if (type.IsValueType)
+                {
+                    il.Emit(OpCodes.Box, type);
                 }
                 il.Emit(OpCodes.Stelem, typeof(object));
             }
             il.Emit(OpCodes.Ldloc_0);
+
+            il.Emit(OpCodes.Ldtoken, method);
 #pragma warning disable CS0618 // PythonDerivedType is for internal use only
             if (method.ReturnType == typeof(void))
             {
-                il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeMethodVoid"));
+                il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod(nameof(InvokeMethodVoid)));
             }
             else
             {
                 il.Emit(OpCodes.Call,
-                    typeof(PythonDerivedType).GetMethod("InvokeMethod").MakeGenericMethod(method.ReturnType));
+                    typeof(PythonDerivedType).GetMethod(nameof(InvokeMethod)).MakeGenericMethod(method.ReturnType));
             }
 #pragma warning restore CS0618 // PythonDerivedType is for internal use only
+            CodeGenerator.GenerateMarshalByRefsBack(il, parameterTypes);
             il.Emit(OpCodes.Ret);
         }
 
@@ -500,35 +509,65 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde
                     argTypes.ToArray());
 
                 ILGenerator il = methodBuilder.GetILGenerator();
+
                 il.DeclareLocal(typeof(object[]));
+                il.DeclareLocal(typeof(RuntimeMethodHandle));
+
+                // this
                 il.Emit(OpCodes.Ldarg_0);
+
+                // Python method to call
                 il.Emit(OpCodes.Ldstr, methodName);
+
+                // original method name
                 il.Emit(OpCodes.Ldnull); // don't fall back to the base type's method
+
+                // create args array
                 il.Emit(OpCodes.Ldc_I4, argTypes.Count);
                 il.Emit(OpCodes.Newarr, typeof(object));
                 il.Emit(OpCodes.Stloc_0);
+
+                // fill args array
                 for (var i = 0; i < argTypes.Count; ++i)
                 {
                     il.Emit(OpCodes.Ldloc_0);
                     il.Emit(OpCodes.Ldc_I4, i);
                     il.Emit(OpCodes.Ldarg, i + 1);
-                    if (argTypes[i].IsValueType)
+                    var type = argTypes[i];
+                    if (type.IsByRef)
                     {
-                        il.Emit(OpCodes.Box, argTypes[i]);
+                        type = type.GetElementType();
+                        il.Emit(OpCodes.Ldobj, type);
+                    }
+                    if (type.IsValueType)
+                    {
+                        il.Emit(OpCodes.Box, type);
                     }
                     il.Emit(OpCodes.Stelem, typeof(object));
                 }
+
+                // args array
                 il.Emit(OpCodes.Ldloc_0);
+
+                // method handle for the base method is null
+                il.Emit(OpCodes.Ldloca_S, 1);
+                il.Emit(OpCodes.Initobj, typeof(RuntimeMethodHandle));
+                il.Emit(OpCodes.Ldloc_1);
 #pragma warning disable CS0618 // PythonDerivedType is for internal use only
+
+                // invoke the method
                 if (returnType == typeof(void))
                 {
-                    il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod("InvokeMethodVoid"));
+                    il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod(nameof(InvokeMethodVoid)));
                 }
                 else
                 {
                     il.Emit(OpCodes.Call,
-                        typeof(PythonDerivedType).GetMethod("InvokeMethod").MakeGenericMethod(returnType));
+                        typeof(PythonDerivedType).GetMethod(nameof(InvokeMethod)).MakeGenericMethod(returnType));
                 }
+
+                CodeGenerator.GenerateMarshalByRefsBack(il, argTypes);
+
 #pragma warning restore CS0618 // PythonDerivedType is for internal use only
                 il.Emit(OpCodes.Ret);
             }
@@ -672,7 +711,8 @@ public class PythonDerivedType
         /// method binding (i.e. it has been overridden in the derived python
         /// class) it calls it, otherwise it calls the base method.
         /// </summary>
-        public static T? InvokeMethod<T>(IPythonDerivedType obj, string methodName, string origMethodName, object[] args)
+        public static T? InvokeMethod<T>(IPythonDerivedType obj, string methodName, string origMethodName,
+            object[] args, RuntimeMethodHandle methodHandle)
         {
             var self = GetPyObj(obj);
 
@@ -698,8 +738,10 @@ public class PythonDerivedType
                             }
 
                             PyObject py_result = method.Invoke(pyargs);
-                            disposeList.Add(py_result);
-                            return py_result.As<T>();
+                            PyTuple? result_tuple = MarshalByRefsBack(args, methodHandle, py_result, outsOffset: 1);
+                            return result_tuple is not null
+                                ? result_tuple[0].As<T>()
+                                : py_result.As<T>();
                         }
                     }
                 }
@@ -726,7 +768,7 @@ public class PythonDerivedType
         }
 
         public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, string origMethodName,
-            object[] args)
+            object?[] args, RuntimeMethodHandle methodHandle)
         {
             var self = GetPyObj(obj);
             if (null != self.Ref)
@@ -736,8 +778,7 @@ public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, s
                 try
                 {
                     using var pyself = new PyObject(self.CheckRun());
-                    PyObject method = pyself.GetAttr(methodName, Runtime.None);
-                    disposeList.Add(method);
+                    using PyObject method = pyself.GetAttr(methodName, Runtime.None);
                     if (method.Reference != Runtime.None)
                     {
                         // if the method hasn't been overridden then it will be a managed object
@@ -752,7 +793,7 @@ public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, s
                             }
 
                             PyObject py_result = method.Invoke(pyargs);
-                            disposeList.Add(py_result);
+                            MarshalByRefsBack(args, methodHandle, py_result, outsOffset: 0);
                             return;
                         }
                     }
@@ -779,6 +820,44 @@ public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, s
                 args);
         }
 
+        /// <summary>
+        /// If the method has byref arguments, reinterprets Python return value
+        /// as a tuple of new values for those arguments, and updates corresponding
+        /// elements of <paramref name="args"/> array.
+        /// </summary>
+        private static PyTuple? MarshalByRefsBack(object?[] args, RuntimeMethodHandle methodHandle, PyObject pyResult, int outsOffset)
+        {
+            if (methodHandle == default) return null;
+
+            var originalMethod = MethodBase.GetMethodFromHandle(methodHandle);
+            var parameters = originalMethod.GetParameters();
+            PyTuple? outs = null;
+            int byrefIndex = 0;
+            for (int i = 0; i < parameters.Length; ++i)
+            {
+                Type type = parameters[i].ParameterType;
+                if (!type.IsByRef)
+                {
+                    continue;
+                }
+
+                type = type.GetElementType();
+
+                if (outs is null)
+                {
+                    outs = new PyTuple(pyResult);
+                    pyResult.Dispose();
+                }
+
+                args[i] = outs[byrefIndex + outsOffset].AsManagedObject(type);
+                byrefIndex++;
+            }
+            if (byrefIndex > 0 && outs!.Length() > byrefIndex + outsOffset)
+                throw new ArgumentException("Too many output parameters");
+
+            return outs;
+        }
+
         public static T? InvokeGetProperty<T>(IPythonDerivedType obj, string propertyName)
         {
             var self = GetPyObj(obj);

diff --git a/src/runtime/codegenerator.cs b/src/runtime/codegenerator.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Reflection;
 using System.Reflection.Emit;
 using System.Threading;
@@ -42,5 +43,39 @@ internal TypeBuilder DefineType(string name, Type basetype)
             var attrs = TypeAttributes.Public;
             return mBuilder.DefineType(name, attrs, basetype);
         }
+
+        /// <summary>
+        /// Generates code, that copies potentially modified objects in args array
+        /// back to the corresponding byref arguments
+        /// </summary>
+        internal static void GenerateMarshalByRefsBack(ILGenerator il, IReadOnlyList<Type> argTypes)
+        {
+            // assumes argument array is in loc_0
+            for (int i = 0; i < argTypes.Count; ++i)
+            {
+                var type = argTypes[i];
+                if (type.IsByRef)
+                {
+                    type = type.GetElementType();
+
+                    il.Emit(OpCodes.Ldarg, i + 1); // for stobj/stind later at the end
+
+                    il.Emit(OpCodes.Ldloc_0);
+                    il.Emit(OpCodes.Ldc_I4, i);
+                    il.Emit(OpCodes.Ldelem_Ref);
+
+                    if (type.IsValueType)
+                    {
+                        il.Emit(OpCodes.Unbox_Any, type);
+                        il.Emit(OpCodes.Stobj, type);
+                    }
+                    else
+                    {
+                        il.Emit(OpCodes.Castclass, type);
+                        il.Emit(OpCodes.Stind_Ref);
+                    }
+                }
+            }
+        }
     }
 }
diff --git a/src/runtime/delegatemanager.cs b/src/runtime/delegatemanager.cs
@@ -135,28 +135,7 @@ private Type GetDispatcher(Type dtype)
             if (anyByRef)
             {
                 // Dispatch() will have modified elements of the args list that correspond to out parameters.
-                for (var c = 0; c < signature.Length; c++)
-                {
-                    Type t = signature[c];
-                    if (t.IsByRef)
-                    {
-                        t = t.GetElementType();
-                        // *arg = args[c]
-                        il.Emit(OpCodes.Ldarg_S, (byte)(c + 1));
-                        il.Emit(OpCodes.Ldloc_0);
-                        il.Emit(OpCodes.Ldc_I4, c);
-                        il.Emit(OpCodes.Ldelem_Ref);
-                        if (t.IsValueType)
-                        {
-                            il.Emit(OpCodes.Unbox_Any, t);
-                            il.Emit(OpCodes.Stobj, t);
-                        }
-                        else
-                        {
-                            il.Emit(OpCodes.Stind_Ref);
-                        }
-                    }
-                }
+                CodeGenerator.GenerateMarshalByRefsBack(il, signature);
             }
 
             if (method.ReturnType == voidtype)

diff --git a/src/testing/interfacetest.cs b/src/testing/interfacetest.cs
@@ -79,4 +79,18 @@ private interface IPrivate
         {
         }
     }
+
+    public interface IOutArg
+    {
+        string MyMethod_Out(string name, out int index);
+    }
+
+    public class OutArgCaller
+    {
+        public static int CallMyMethod_Out(IOutArg myInterface)
+        {
+            myInterface.MyMethod_Out("myclient", out int index);
+            return index;
+        }
+    }
 }
diff --git a/tests/test_interface.py b/tests/test_interface.py
@@ -93,6 +93,20 @@ def test_interface_object_returned_through_out_param():
 
     assert hello2.SayHello() == 'hello 2'
 
+def test_interface_out_param_python_impl():
+    from Python.Test import IOutArg, OutArgCaller
+
+    class MyOutImpl(IOutArg):
+        __namespace__ = "Python.Test"
+
+        def MyMethod_Out(self, name, index):
+            other_index = 101
+            return ('MyName', other_index)
+
+    py_impl = MyOutImpl()
+
+    assert 101 == OutArgCaller.CallMyMethod_Out(py_impl)
+
 
 def test_null_interface_object_returned():
     """Test None is used also for methods with interface return types"""