From ab531f85a1bdc32744676f7fabb894ee481c5da2 Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Wed, 27 Apr 2022 22:47:25 +0200 Subject: [PATCH 01/14] 1776 Inherit Generic Virtual Method Bug: Unit Test Added a unit test which consists ofa python class inheriting from a C# class which has a virtual generic method. In this version, this causes a InvalidProgramException to be thrown during class creation. --- src/testing/generictest.cs | 8 ++++++++ tests/test_subclass.py | 8 +++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/testing/generictest.cs b/src/testing/generictest.cs index 238435811..b333910c2 100644 --- a/src/testing/generictest.cs +++ b/src/testing/generictest.cs @@ -136,4 +136,12 @@ public static T[] EchoRange(T[] items) return items; } } + + public abstract class GenericVirtualMethodTest + { + public virtual Q VirtMethod(Q arg1) + { + return arg1; + } + } } diff --git a/tests/test_subclass.py b/tests/test_subclass.py index fa82c3663..3d4c1d40d 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -9,7 +9,7 @@ import System import pytest from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, - FunctionsTest) + FunctionsTest, GenericVirtualMethodTest) from System.Collections.Generic import List @@ -264,6 +264,12 @@ class TestX(System.Object): t = TestX() assert t.q == 1 +def test_virtual_generic_method(): + class OverloadingSubclass(GenericVirtualMethodTest): + __namespace__ = "test_virtual_generic_method_cls" + obj = OverloadingSubclass() + assert obj.VirtMethod[int](5) == 5 + def test_construction_from_clr(): import clr calls = [] From bfcf1d1b66f7d8d2d63989d506460713c91f8bf5 Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Thu, 28 Apr 2022 08:48:08 +0200 Subject: [PATCH 02/14] 1776 Inherit Generic Virtual Method Bug: Fix Previously an exception was thrown during class creation if the python class inherited from a class with a virtual generic method. This has been fixed by ignoring generic methods when creating overloads in the generated class. Note, this means that generic virtual methods cannot be overridden in python code. --- src/runtime/Types/ClassDerived.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 02288faee..07367a04a 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -219,8 +219,9 @@ internal static Type CreateDerivedType(string name, var virtualMethods = new HashSet(); foreach (MethodInfo method in methods) { - if (!method.Attributes.HasFlag(MethodAttributes.Virtual) | - method.Attributes.HasFlag(MethodAttributes.Final)) + if (!method.Attributes.HasFlag(MethodAttributes.Virtual) + || method.Attributes.HasFlag(MethodAttributes.Final) + || method.IsGenericMethod) { continue; } From eb4ce37eb5d666a48c7119420ef0e5e272c51d02 Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Fri, 29 Apr 2022 08:14:26 +0200 Subject: [PATCH 03/14] 1774-ClassWithoutnamespace - Changed the behavior so that a .NET is always created when the base type is also a .NET type. --- src/runtime/Types/MetaType.cs | 10 +++++++++- tests/test_subclass.py | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/runtime/Types/MetaType.cs b/src/runtime/Types/MetaType.cs index 5b59f5139..db1a3ff2e 100644 --- a/src/runtime/Types/MetaType.cs +++ b/src/runtime/Types/MetaType.cs @@ -128,10 +128,18 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, if (null != dict) { using var clsDict = new PyDict(dict); - if (clsDict.HasKey("__assembly__") || clsDict.HasKey("__namespace__")) + + if (clsDict.HasKey("__assembly__") || clsDict.HasKey("__namespace__") + || (cb.type.Valid && cb.type.Value != null && cb.type.Value.GetConstructor(Array.Empty()) != null)) { + if (!clsDict.HasKey("__namespace__")) + { + clsDict["__namespace__"] = + (clsDict["__module__"].ToString()).ToPython(); + } return TypeManager.CreateSubType(name, base_type, clsDict); } + } // otherwise just create a basic type without reflecting back into the managed side. diff --git a/tests/test_subclass.py b/tests/test_subclass.py index 3d4c1d40d..8b15d31ff 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -267,6 +267,8 @@ class TestX(System.Object): def test_virtual_generic_method(): class OverloadingSubclass(GenericVirtualMethodTest): __namespace__ = "test_virtual_generic_method_cls" + class OverloadingSubclass2(OverloadingSubclass): + pass obj = OverloadingSubclass() assert obj.VirtMethod[int](5) == 5 From a2c80cd7f567c42b2dc480be53a7fe2619e5fe3b Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Tue, 3 May 2022 18:52:33 +0200 Subject: [PATCH 04/14] better support for multiple inheritance --- src/python_tests_runner/PythonTestRunner.cs | 4 ++ src/runtime/Runtime.cs | 6 ++ src/runtime/StateSerialization/MaybeType.cs | 4 +- src/runtime/TypeManager.cs | 7 ++- src/runtime/Types/ClassDerived.cs | 7 ++- src/runtime/Types/MetaType.cs | 61 +++++++++++++++------ src/runtime/Types/ReflectedClrType.cs | 3 +- src/testing/generictest.cs | 10 ++-- src/testing/subclasstest.cs | 21 +++++++ tests/test_subclass.py | 24 +++++++- 10 files changed, 117 insertions(+), 30 deletions(-) diff --git a/src/python_tests_runner/PythonTestRunner.cs b/src/python_tests_runner/PythonTestRunner.cs index 05298997b..98b2fbe68 100644 --- a/src/python_tests_runner/PythonTestRunner.cs +++ b/src/python_tests_runner/PythonTestRunner.cs @@ -17,6 +17,8 @@ public class PythonTestRunner [OneTimeSetUp] public void SetUp() { + Python.Runtime.Runtime.PythonDLL = + "/Library/Frameworks/Python.framework/Versions/3.10/lib/libpython3.10.dylib"; PythonEngine.Initialize(); } @@ -35,6 +37,8 @@ static IEnumerable PythonTestCases() // Add the test that you want to debug here. yield return new[] { "test_indexer", "test_boolean_indexer" }; yield return new[] { "test_delegate", "test_bool_delegate" }; + yield return new[] { "test_subclass", "test_virtual_generic_method" }; + yield return new[] { "test_subclass", "test_interface_and_class_impl2" }; } /// diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 7110f3cb0..f157f85ae 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -1833,6 +1833,12 @@ internal static void SetNoSiteFlag() return *Delegates.Py_NoSiteFlag; }); } + + internal static uint PyTuple_GetSize(BorrowedReference tuple) + { + IntPtr r = Delegates.PyTuple_Size(tuple); + return (uint)r.ToInt32(); + } } internal class BadPythonDllException : MissingMethodException diff --git a/src/runtime/StateSerialization/MaybeType.cs b/src/runtime/StateSerialization/MaybeType.cs index f3c96e369..549c5f29e 100644 --- a/src/runtime/StateSerialization/MaybeType.cs +++ b/src/runtime/StateSerialization/MaybeType.cs @@ -15,7 +15,6 @@ internal struct MaybeType : ISerializable const string SerializationName = "n"; readonly string name; readonly Type type; - public string DeletedMessage { get @@ -38,6 +37,7 @@ public Type Value public string Name => name; public bool Valid => type != null; + public Type ValueOrNull => type; public override string ToString() { @@ -61,4 +61,4 @@ public void GetObjectData(SerializationInfo serializationInfo, StreamingContext serializationInfo.AddValue(SerializationName, name); } } -} \ No newline at end of file +} diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 217b4820e..1099d6430 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -374,7 +374,7 @@ static PyTuple GetBaseTypeTuple(Type clrType) return new PyTuple(bases); } - internal static NewReference CreateSubType(BorrowedReference py_name, BorrowedReference py_base_type, BorrowedReference dictRef) + internal static NewReference CreateSubType(BorrowedReference py_name, IEnumerable py_base_type, IEnumerable interfaces, BorrowedReference dictRef) { // Utility to create a subtype of a managed type with the ability for the // a python subtype able to override the managed implementation @@ -415,9 +415,10 @@ internal static NewReference CreateSubType(BorrowedReference py_name, BorrowedRe } // create the new managed type subclassing the base managed type - if (ManagedType.GetManagedObject(py_base_type) is ClassBase baseClass) + var baseClass = py_base_type.FirstOrDefault(); + if (null == baseClass) { - return ReflectedClrType.CreateSubclass(baseClass, name, + return ReflectedClrType.CreateSubclass(baseClass, interfaces, name, ns: (string?)namespaceStr, assembly: (string?)assembly, dict: dictRef); diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 07367a04a..a45beb78f 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -144,6 +144,7 @@ internal static NewReference ToPython(IPythonDerivedType obj) /// internal static Type CreateDerivedType(string name, Type baseType, + IEnumerable interfaces2, BorrowedReference py_dict, string? namespaceStr, string? assemblyName, @@ -163,8 +164,8 @@ internal static Type CreateDerivedType(string name, ModuleBuilder moduleBuilder = GetModuleBuilder(assemblyName, moduleName); Type baseClass = baseType; - var interfaces = new List { typeof(IPythonDerivedType) }; - + var interfaces = new HashSet { typeof(IPythonDerivedType) }; + foreach(var t in interfaces2) interfaces.Add(t); // if the base type is an interface then use System.Object as the base class // and add the base type to the list of interfaces this new class will implement. if (baseType.IsInterface) @@ -215,7 +216,7 @@ internal static Type CreateDerivedType(string name, } // override any virtual methods not already overridden by the properties above - MethodInfo[] methods = baseType.GetMethods(); + var methods = baseType.GetMethods().Concat(interfaces.SelectMany(x => x.GetMethods())); var virtualMethods = new HashSet(); foreach (MethodInfo method in methods) { diff --git a/src/runtime/Types/MetaType.cs b/src/runtime/Types/MetaType.cs index db1a3ff2e..459796ebf 100644 --- a/src/runtime/Types/MetaType.cs +++ b/src/runtime/Types/MetaType.cs @@ -1,5 +1,8 @@ using System; +using System.Collections.Generic; using System.Diagnostics; +using System.Linq; +using System.Reflection; using System.Runtime.InteropServices; using System.Runtime.Serialization; @@ -84,36 +87,58 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, // That type must itself have a managed implementation. We check // that by making sure its metatype is the CLR metatype. - if (Runtime.PyTuple_Size(bases) != 1) + + List interfaces = new List(); + List baseType = new List(); + + var cnt = Runtime.PyTuple_GetSize(bases); + + for (uint i = 0; i < cnt; i++) + { + var base_type2 = Runtime.PyTuple_GetItem(bases, (int)i); + var cb2 = (ClassBase) GetManagedObject(base_type2); + if (cb2 != null) + { + if (cb2.type.Valid && cb2.type.Value.IsInterface) + interfaces.Add(cb2.type.Value); + else baseType.Add(cb2); + } + } + + if (baseType.Count == 0) + { + baseType.Add(new ClassBase(typeof(object))); + } + + + if (baseType.Count > 1) { return Exceptions.RaiseTypeError("cannot use multiple inheritance with managed classes"); } - BorrowedReference base_type = Runtime.PyTuple_GetItem(bases, 0); - BorrowedReference mt = Runtime.PyObject_TYPE(base_type); + /* + BorrowedReference mt = Runtime.PyObject_TYPE(baseType); if (!(mt == PyCLRMetaType || mt == Runtime.PyTypeType)) { return Exceptions.RaiseTypeError("invalid metatype"); - } + }*/ // Ensure that the reflected type is appropriate for subclassing, // disallowing subclassing of delegates, enums and array types. - if (GetManagedObject(base_type) is ClassBase cb) + var cb = baseType.First(); + try { - try + if (!cb.CanSubclass()) { - if (!cb.CanSubclass()) - { - return Exceptions.RaiseTypeError("delegates, enums and array types cannot be subclassed"); - } - } - catch (SerializationException) - { - return Exceptions.RaiseTypeError($"Underlying C# Base class {cb.type} has been deleted"); + return Exceptions.RaiseTypeError("delegates, enums and array types cannot be subclassed"); } } + catch (SerializationException) + { + return Exceptions.RaiseTypeError($"Underlying C# Base class {cb.type} has been deleted"); + } BorrowedReference slots = Runtime.PyDict_GetItem(dict, PyIdentifier.__slots__); if (slots != null) @@ -127,21 +152,25 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, // into python. if (null != dict) { + var btt = baseType.FirstOrDefault().type.ValueOrNull; + var ctor = btt?.GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .FirstOrDefault(x => x.GetParameters().Any() == false); using var clsDict = new PyDict(dict); if (clsDict.HasKey("__assembly__") || clsDict.HasKey("__namespace__") - || (cb.type.Valid && cb.type.Value != null && cb.type.Value.GetConstructor(Array.Empty()) != null)) + || (ctor != null)) { if (!clsDict.HasKey("__namespace__")) { clsDict["__namespace__"] = (clsDict["__module__"].ToString()).ToPython(); } - return TypeManager.CreateSubType(name, base_type, clsDict); + return TypeManager.CreateSubType(name, baseType, interfaces, clsDict); } } + var base_type = Runtime.PyTuple_GetItem(bases, 0); // otherwise just create a basic type without reflecting back into the managed side. IntPtr func = Util.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_new); NewReference type = NativeCall.Call_3(func, tp, args, kw); diff --git a/src/runtime/Types/ReflectedClrType.cs b/src/runtime/Types/ReflectedClrType.cs index d3d89bdb8..41d568ac4 100644 --- a/src/runtime/Types/ReflectedClrType.cs +++ b/src/runtime/Types/ReflectedClrType.cs @@ -68,7 +68,7 @@ internal void Restore(ClassBase cb) TypeManager.InitializeClass(this, cb, cb.type.Value); } - internal static NewReference CreateSubclass(ClassBase baseClass, + internal static NewReference CreateSubclass(ClassBase baseClass, IEnumerable interfaces, string name, string? assembly, string? ns, BorrowedReference dict) { @@ -76,6 +76,7 @@ internal static NewReference CreateSubclass(ClassBase baseClass, { Type subType = ClassDerivedObject.CreateDerivedType(name, baseClass.type.Value, + interfaces, dict, ns, assembly); diff --git a/src/testing/generictest.cs b/src/testing/generictest.cs index b333910c2..a78fd5104 100644 --- a/src/testing/generictest.cs +++ b/src/testing/generictest.cs @@ -95,6 +95,11 @@ public string Overloaded(int arg1, int arg2, string arg3) { return arg3; } + + public virtual Q VirtualOverloaded(Q arg) + { + return arg; + } } public class GenericStaticMethodTest @@ -118,10 +123,7 @@ public static Q Overloaded(Q arg) return arg; } - public static U Overloaded(Q arg1, U arg2) - { - return arg2; - } + public static string Overloaded(int arg1, int arg2, string arg3) { diff --git a/src/testing/subclasstest.cs b/src/testing/subclasstest.cs index ab0b73368..3ad34c6be 100644 --- a/src/testing/subclasstest.cs +++ b/src/testing/subclasstest.cs @@ -124,4 +124,25 @@ public static int test_event(IInterfaceTest x, int value) return et.value; } } + + public interface ISimpleInterface + { + bool Ok(); + } + + public class SimpleClass + { + + public static void TestObject(object obj) + { + if (obj is ISimpleInterface) + { + + } + else + { + throw new Exception(); + } + } + } } diff --git a/tests/test_subclass.py b/tests/test_subclass.py index 8b15d31ff..53e48f159 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -9,7 +9,7 @@ import System import pytest from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, - FunctionsTest, GenericVirtualMethodTest) + FunctionsTest, GenericVirtualMethodTest, ISimpleInterface, SimpleClass) from System.Collections.Generic import List @@ -272,6 +272,28 @@ class OverloadingSubclass2(OverloadingSubclass): obj = OverloadingSubclass() assert obj.VirtMethod[int](5) == 5 +def test_interface_and_class_impl(): + class OverloadingSubclass(GenericVirtualMethodTest): + __namespace__ = "test_virtual_generic_method_cls" + class OverloadingSubclass2(OverloadingSubclass): + pass + obj = OverloadingSubclass() + assert obj.VirtMethod[int](5) == 5 + +def test_interface_and_class_impl2(): + class DualSubClass(ISimpleInterface, SimpleClass): + def Ok(self): + return True + class DualSubClass2(ISimpleInterface): + def Ok(self): + return True + + obj = DualSubClass() + SimpleClass.TestObject(obj) + obj = DualSubClass2() + SimpleClass.TestObject(obj) + + def test_construction_from_clr(): import clr calls = [] From 65c2c008834bc4380aad322fb4a72ef7c5a68752 Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Tue, 3 May 2022 22:46:03 +0200 Subject: [PATCH 05/14] Fixed issue with protected constructors and classes with no constructor --- src/runtime/Types/ClassDerived.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index a45beb78f..779bd0262 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -188,12 +188,17 @@ internal static Type CreateDerivedType(string name, FieldAttributes.Private); // override any constructors - ConstructorInfo[] constructors = baseClass.GetConstructors(); + ConstructorInfo[] constructors = baseClass.GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); foreach (ConstructorInfo ctor in constructors) { AddConstructor(ctor, baseType, typeBuilder); } + if (constructors.Length == 0) + { + AddConstructor(null, baseType, typeBuilder); + } + // Override any properties explicitly overridden in python var pyProperties = new HashSet(); if (py_dict != null && Runtime.PyDict_Check(py_dict)) @@ -303,7 +308,7 @@ internal static Type CreateDerivedType(string name, /// TypeBuilder for the new type the ctor is to be added to private static void AddConstructor(ConstructorInfo ctor, Type baseType, TypeBuilder typeBuilder) { - ParameterInfo[] parameters = ctor.GetParameters(); + ParameterInfo[] parameters = ctor?.GetParameters() ?? Array.Empty(); Type[] parameterTypes = (from param in parameters select param.ParameterType).ToArray(); // create a method for calling the original constructor @@ -322,14 +327,15 @@ private static void AddConstructor(ConstructorInfo ctor, Type baseType, TypeBuil { il.Emit(OpCodes.Ldarg, i + 1); } - il.Emit(OpCodes.Call, ctor); + if(ctor != null) + il.Emit(OpCodes.Call, ctor); il.Emit(OpCodes.Ret); // override the original method with a new one that dispatches to python ConstructorBuilder cb = typeBuilder.DefineConstructor(MethodAttributes.Public | MethodAttributes.ReuseSlot | MethodAttributes.HideBySig, - ctor.CallingConvention, + ctor?.CallingConvention ?? CallingConventions.Any, parameterTypes); il = cb.GetILGenerator(); il.DeclareLocal(typeof(object[])); From 28f47170d0b6ecce383076060b956cbf3705f360 Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Thu, 5 May 2022 19:26:30 +0200 Subject: [PATCH 06/14] Added supporr for class/property/method attributes. - Added shortened Attribute aliases which just generates tuples. - More general CLR base class support - creating a class instance from C# is now more similar to creating one from python. - Added attribute decorator to clr.py. - Added testing for the various possibilities --- src/python_tests_runner/PythonTestRunner.cs | 3 + src/runtime/Resources/clr.py | 67 +++++- src/runtime/Types/ClassDerived.cs | 231 +++++++++++++++++++- src/runtime/Types/ModuleObject.cs | 17 ++ src/testing/subclasstest.cs | 71 +++++- tests/test_subclass.py | 81 ++++++- 6 files changed, 454 insertions(+), 16 deletions(-) diff --git a/src/python_tests_runner/PythonTestRunner.cs b/src/python_tests_runner/PythonTestRunner.cs index 98b2fbe68..b4f030fd7 100644 --- a/src/python_tests_runner/PythonTestRunner.cs +++ b/src/python_tests_runner/PythonTestRunner.cs @@ -39,6 +39,9 @@ static IEnumerable PythonTestCases() yield return new[] { "test_delegate", "test_bool_delegate" }; yield return new[] { "test_subclass", "test_virtual_generic_method" }; yield return new[] { "test_subclass", "test_interface_and_class_impl2" }; + yield return new[] { "test_subclass", "test_class_with_attributes" }; + yield return new[] { "test_subclass", "test_class_with_advanced_attribute" }; + yield return new[] { "test_subclass", "test_more_subclasses" }; } /// diff --git a/src/runtime/Resources/clr.py b/src/runtime/Resources/clr.py index d4330a4d5..a5ff60370 100644 --- a/src/runtime/Resources/clr.py +++ b/src/runtime/Resources/clr.py @@ -21,16 +21,18 @@ def test(self): string z = x.test; // calls into python and returns "x" """ - def __init__(self, type_, fget=None, fset=None): + def __init__(self, type_, fget=None, fset=None, attributes = []): self.__name__ = getattr(fget, "__name__", None) self._clr_property_type_ = type_ self.fget = fget self.fset = fset - + self._clr_attributes_ = attributes def __call__(self, fget): - return self.__class__(self._clr_property_type_, + self.__class__(self._clr_property_type_, fget=fget, - fset=self.fset) + fset=self.fset, + attributes = self._clr_attributes_) + def setter(self, fset): self.fset = fset @@ -47,8 +49,32 @@ def __set__(self, instance, value): if not self.fset: raise AttributeError("%s is read-only" % self.__name__) return self.fset.__get__(instance, None)(value) + def add_attribute(self, attribute): + self._clr_attributes_.append(attribute) + return self +class property(object): + def __init__(self, type, default): + import weakref + self._clr_property_type_ = type + self.default = default + self.values = weakref.WeakKeyDictionary() + self._clr_attributes_ = [] + self.fget = 1 + self.fset = 1 + def __get__(self, instance, owner): + v = self.values.get(instance, self.default) + return v + def __set__(self, instance, value): + self.values[instance] = value + def add_attribute(self, attribute): + self._clr_attributes_.append(attribute) + return self + def __call__(self, type, default): + self2 = self.__class__(self._clr_property_type_, type, default) + self2._clr_attributes_ = self._clr_attributes_ + return self2 class clrmethod(object): """ Method decorator for exposing python methods to .NET. @@ -67,18 +93,47 @@ def test(self, x): int z = x.test("hello"); // calls into python and returns len("hello") """ - def __init__(self, return_type, arg_types, clrname=None, func=None): + def __init__(self, return_type = None, arg_types = [], clrname=None, func=None, **kwargs): + if return_type == None: + import System + return_type = System.Void self.__name__ = getattr(func, "__name__", None) self._clr_return_type_ = return_type self._clr_arg_types_ = arg_types self._clr_method_name_ = clrname or self.__name__ self.__func = func + if 'attributes' in kwargs: + self._clr_attributes_ = kwargs["attributes"] + else: + self._clr_attributes_ = [] def __call__(self, func): - return self.__class__(self._clr_return_type_, + self2 = self.__class__(self._clr_return_type_, self._clr_arg_types_, clrname=self._clr_method_name_, func=func) + self2._clr_attributes_ = self._clr_attributes_ + return self2 def __get__(self, instance, owner): return self.__func.__get__(instance, owner) + + def clr_attribute(self, attribute): + self._clr_attributes_.append(attribute) + return self + +class attribute(object): + + def __init__(self, attr, *args, **kwargs): + self.attr = attr + import Python.Runtime + #todo: ensure that attributes only are pushed when @ is used. + #import inspect + #Python.Runtime.PythonDerivedType.Test(inspect.stack()[1].code_context) + + Python.Runtime.PythonDerivedType.PushAttribute(attr) + def __call__(self, x): + import Python.Runtime + if Python.Runtime.PythonDerivedType.AssocAttribute(self.attr, x): + pass + return x diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 779bd0262..6404b7460 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -137,6 +137,112 @@ internal static NewReference ToPython(IPythonDerivedType obj) return result; } + static CustomAttributeBuilder AddAttribute(PyObject attr) + { + // this is a bit complicated because we want to support both unnamed and named arguments + // in C# there is a discrepancy between property setters and named argument wrt attributes + // in python there is no difference. But luckily there is rarely a conflict, so we can treat + // named arguments and named properties or fields the same way. + var tp = new PyTuple(attr); + Type attribute = (Type) tp[0].AsManagedObject(typeof(object)); + + var args = (PyObject[]) tp[1].AsManagedObject(typeof(PyObject[])); + var dict = new PyDict(tp[2]); + var dict2 = new Dictionary(); + foreach (var key in dict.Keys()) + { + var k = key.As(); + dict2[k] = dict[key]; + } + + // todo support kwargs and tupleargs + var allconstructors = attribute.GetConstructors(); + foreach (var constructor in allconstructors) + { + var parameters = constructor.GetParameters(); + List paramValues = new List(); + HashSet accountedFor = new HashSet(); + for (int i = 0; i < parameters.Length; i++) + { + var parameter = parameters[i]; + if (parameter.ParameterType.IsArray && null != parameter.GetCustomAttribute(typeof(ParamArrayAttribute))) + { + int cnt = args.Length - i; + var values = Array.CreateInstance(parameter.ParameterType.GetElementType(), cnt); + for(int j = 0; j < cnt; j++) + values.SetValue(args[i + j], j); + paramValues.Add(values); + break; + } + + PyObject argvalue = null; + if (args.Length <= i && dict2.TryGetValue(parameter.Name, out argvalue)) + { + accountedFor.Add(parameter.Name); + }else if (args.Length <= i && parameter.IsOptional) + { + paramValues.Add(parameter.DefaultValue); + continue; + }else if (args.Length <= i) + { + goto next; + } + else + { + argvalue = args[i]; + } + + var argval = argvalue.AsManagedObject(parameter.ParameterType); + if (parameter.ParameterType.IsAssignableFrom(argval?.GetType() ?? typeof(object))) + paramValues.Add(argval); + else if (parameter.IsOptional) + { + paramValues.Add(parameter.DefaultValue); + } + } + + List namedProperties = new List(); + List namedPropertyValues = new List(); + List namedFields = new List(); + List namedFieldValues = new List(); + + foreach (var key in dict2.Keys.Where(x => accountedFor.Contains(x) == false)) + { + var member = attribute.GetMember(key).FirstOrDefault(); + if (member == null) + goto next; + if (member is PropertyInfo prop) + { + namedProperties.Add(prop); + var argval = dict2[key].AsManagedObject(prop.PropertyType); + if (prop.PropertyType.IsAssignableFrom(argval?.GetType() ?? typeof(object))) + namedPropertyValues.Add(argval); + else + goto next; + } + if (member is FieldInfo field) + { + namedFields.Add(field); + var argval = dict2[key].AsManagedObject(field.FieldType); + if (field.FieldType.IsAssignableFrom(argval?.GetType() ?? typeof(object))) + namedFieldValues.Add(argval); + else + goto next; + } + } + + var cb = new CustomAttributeBuilder(constructor, paramValues.ToArray(), + namedProperties.ToArray(), namedPropertyValues.ToArray(), + namedFields.ToArray(), namedFieldValues.ToArray()); + return cb; + next: ; + } + + return null; + } + + + /// /// Creates a new managed type derived from a base type with any virtual /// methods overridden to call out to python if the associated python @@ -181,11 +287,14 @@ internal static Type CreateDerivedType(string name, // add a field for storing the python object pointer // FIXME: fb not used - FieldBuilder fb = typeBuilder.DefineField(PyObjName, + if (baseClass.GetField(PyObjName, PyObjFlags) == null) + { + FieldBuilder fb = typeBuilder.DefineField(PyObjName, #pragma warning disable CS0618 // Type or member is obsolete. OK for internal use. - typeof(UnsafeReferenceWithRun), + typeof(UnsafeReferenceWithRun), #pragma warning restore CS0618 // Type or member is obsolete - FieldAttributes.Private); + FieldAttributes.Public); + } // override any constructors ConstructorInfo[] constructors = baseClass.GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); @@ -250,6 +359,24 @@ internal static Type CreateDerivedType(string name, if (py_dict != null && Runtime.PyDict_Check(py_dict)) { using var dict = new PyDict(py_dict); + if (dict.HasKey("__clr_attributes__")) + { + var attributes = new PyList(dict["__clr_attributes__"]); + foreach (var attr in attributes) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + typeBuilder.SetCustomAttribute(builder); + } + } + + foreach (var attr in PopAttributes()) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + typeBuilder.SetCustomAttribute(builder); + } + using var keys = dict.Keys(); foreach (PyObject pyKey in keys) { @@ -290,6 +417,7 @@ internal static Type CreateDerivedType(string name, Type type = typeBuilder.CreateType(); + // scan the assembly so the newly added class can be imported Assembly assembly = Assembly.GetAssembly(type); AssemblyManager.ScanAssembly(assembly); @@ -300,6 +428,8 @@ internal static Type CreateDerivedType(string name, return type; } + private static Dictionary pyTypeLookup = new Dictionary(); + /// /// Add a constructor override that calls the python ctor after calling the base type constructor. /// @@ -478,7 +608,8 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde } using var pyReturnType = func.GetAttr("_clr_return_type_"); - using var pyArgTypes = func.GetAttr("_clr_arg_types_"); + usingvar attributes = new PyList(func.GetAttr("_clr_attributes_"))) + using (var pyArgTypes = func.GetAttr("_clr_arg_types_"); using var pyArgTypesIter = PyIter.GetIter(pyArgTypes); var returnType = pyReturnType.AsManagedObject(typeof(Type)) as Type; if (returnType == null) @@ -509,7 +640,27 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde returnType, argTypes.ToArray()); - ILGenerator il = methodBuilder.GetILGenerator(); + foreach (var attr in attributes) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + methodBuilder.SetCustomAttribute(builder); + } + + if (methodAssoc.TryGetValue(func, out var lst)) + { + foreach(var attr in lst) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + methodBuilder.SetCustomAttribute(builder); + } + + methodAssoc.Remove(func); + + } + + ILGenerator il = methodBuilder.GetILGenerator(); il.DeclareLocal(typeof(object[])); il.DeclareLocal(typeof(RuntimeMethodHandle)); @@ -600,6 +751,18 @@ private static void AddPythonProperty(string propertyName, PyObject func, TypeBu PropertyAttributes.None, propertyType, null); + if (func.HasAttr("_clr_attributes_")) + { + using (var attributes = new PyList(func.GetAttr("_clr_attributes_"))) + { + foreach (var attr in attributes) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + propertyBuilder.SetCustomAttribute(builder); + } + } + } if (func.HasAttr("fget")) { @@ -695,8 +858,45 @@ private static ModuleBuilder GetModuleBuilder(string assemblyName, string module [Obsolete(Util.InternalUseOnly)] public class PythonDerivedType { + private static List attributesStack = new List(); + internal static Dictionary> methodAssoc = new Dictionary>(); + public static void PushAttribute(PyObject obj) + { + var tp = new PyTuple(obj); + attributesStack.Add(tp); + } + + public static bool AssocAttribute(PyObject obj, PyObject func) + { + using var _ = Py.GIL(); + var tp = new PyTuple(obj); + for (int i = 0; i < attributesStack.Count; i++) + { + if (Equals(tp, attributesStack[i])) + { + attributesStack.RemoveAt(i); + if (!methodAssoc.TryGetValue(func, out var lst)) + { + lst = methodAssoc[func] = new List(); + } + lst.Add(tp); + return true; + } + } + return false; + + } + + + public static IEnumerable PopAttributes() + { + if (attributesStack.Count == 0) return Array.Empty(); + var attrs = attributesStack; + attributesStack = new List(); + return attrs; + } internal const string PyObjName = "__pyobj__"; - internal const BindingFlags PyObjFlags = BindingFlags.Instance | BindingFlags.NonPublic; + internal const BindingFlags PyObjFlags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.FlattenHierarchy; /// /// This is the implementation of the overridden methods in the derived @@ -896,6 +1096,7 @@ public static void InvokeSetProperty(IPythonDerivedType obj, string propertyN } } + static private PyObject pin; public static void InvokeCtor(IPythonDerivedType obj, string origCtorName, object[] args) { var selfRef = GetPyObj(obj); @@ -906,8 +1107,22 @@ public static void InvokeCtor(IPythonDerivedType obj, string origCtorName, objec // In the end we decrement the python object's reference count. // This doesn't actually destroy the object, it just sets the reference to this object // to be a weak reference and it will be destroyed when the C# object is destroyed. - using var self = CLRObject.GetReference(obj, obj.GetType()); - SetPyObj(obj, self.Borrow()); + + PyType cc = ClassManager.GetClass(obj.GetType()); + var args2 = new PyObject[args.Length]; + for (int i = 0; i < args.Length; i++) + args2[i] = args[i].ToPython(); + // create an instance of the class and steal the reference. + using (var obj2 = cc.Invoke(args2, null)) + { + // somehow if this is not done this object never gets a reference count + // and things breaks later. + // I am not sure what it does though. + GCHandle gc = GCHandle.Alloc(obj); + // hand over the reference. + var py = obj2.NewReferenceOrNull(); + SetPyObj(obj, py.Borrow()); + } } // call the base constructor diff --git a/src/runtime/Types/ModuleObject.cs b/src/runtime/Types/ModuleObject.cs index f641b393e..af8c28ebd 100644 --- a/src/runtime/Types/ModuleObject.cs +++ b/src/runtime/Types/ModuleObject.cs @@ -132,6 +132,23 @@ public NewReference GetAttribute(string name, bool guess) return new NewReference(c); } + // attribute names + // for attributes without the Attribute suffix, create an attribute builder. + var qname2 = qname + "Attribute"; + var type2 = AssemblyManager.LookupTypes(qname2).FirstOrDefault(t => t.IsPublic); + if (type2 != null) + { + var str = "def {2}attrbuilder(*args, **kwargs):\n" + + " import {1}\n" + + " return ({0}, args, kwargs)\n"; + str = string.Format(str, qname2, _namespace, name); + PythonEngine.Exec(str); + var obj = PythonEngine.Eval(name + "attrbuilder"); + var o = obj.NewReferenceOrNull(); + this.StoreAttribute(name, o.Borrow()); + return new NewReference(o); + } + // We didn't find the name, so we may need to see if there is a // generic type with this base name. If so, we'll go ahead and // return it. Note that we store the mapping of the unmangled diff --git a/src/testing/subclasstest.cs b/src/testing/subclasstest.cs index 3ad34c6be..989929ea3 100644 --- a/src/testing/subclasstest.cs +++ b/src/testing/subclasstest.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Threading; namespace Python.Test { @@ -130,13 +131,42 @@ public interface ISimpleInterface bool Ok(); } + public class TestAttributeAttribute: Attribute + { + public int X { get; set; } + public int Y { get; set; } + public string Z { get; set; } + public string W { get; set; } + public TestAttributeAttribute(int x, int y, string z = "x") + { + X = x; + Y = y; + Z = z; + + } + } + + public class SimpleClass { + public bool Initialized; + + public SimpleClass() + { + Initialized = true; + } + private int counter = 0; + public virtual int IncrementThing() + { + return ++counter; + } public static void TestObject(object obj) { - if (obj is ISimpleInterface) + if (obj is ISimpleInterface si) { + if (!si.Ok()) + throw new Exception(); } else @@ -144,5 +174,44 @@ public static void TestObject(object obj) throw new Exception(); } } + public static void TestObjectProperty(object obj, string prop, double newval) + { + obj.GetType().GetProperty(prop).SetValue(obj, newval); + var val = obj.GetType().GetProperty(prop).GetValue(obj); + if (!Equals(newval, val)) + throw new Exception(); + } + + private static SimpleClass objStore; + public static void Test1(SimpleClass obj) + { + objStore = obj; + int x = obj.IncrementThing(); + } + + public static void Test2() + { + GC.Collect(); + + var threads = new Thread[20]; + for(int i = 0; i < threads.Length; i++) + threads[i] = new Thread(() => TestObjectProperty(objStore, "X", 10.0)); + for (int i = 0; i < threads.Length; i++) + threads[i].Start(); + for (int i = 0; i < threads.Length; i++) + threads[i].Join(); + } + + public static object InvokeCtor(Type t) + { + var obj = Activator.CreateInstance(t); + return obj; + } + + public static void Pause() + { + + } + } } diff --git a/tests/test_subclass.py b/tests/test_subclass.py index 53e48f159..828435183 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -7,9 +7,12 @@ """Test sub-classing managed types""" import System +from System import (Console, Attribute, Double) +from System.Diagnostics import (DebuggerDisplay, DebuggerDisplayAttribute, Debug) +from System.ComponentModel import (Browsable, BrowsableAttribute) import pytest from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, - FunctionsTest, GenericVirtualMethodTest, ISimpleInterface, SimpleClass) + FunctionsTest, GenericVirtualMethodTest, ISimpleInterface, SimpleClass, TestAttribute) from System.Collections.Generic import List @@ -293,6 +296,82 @@ def Ok(self): obj = DualSubClass2() SimpleClass.TestObject(obj) +def test_class_with_attributes(): + import clr + @clr.attribute(Browsable(False)) + class ClassWithAttributes(ISimpleInterface, SimpleClass): + __clr_attributes__ = [DebuggerDisplay("X: {X}")] + @clr.attribute(Browsable(True)) + def Ok(self): + return True + @clr.attribute(Browsable(True)) + @clr.clrmethod(int, [int]) + def Method1(x): + return x + + X = clr.property(Double, 1.0).add_attribute(DebuggerDisplay("Asd")) + obj = ClassWithAttributes() + tp = obj.GetType() + founddisplay = 0 + foundbrowsable = 0 + for attr in Attribute.GetCustomAttributes(tp): + if isinstance(attr, DebuggerDisplayAttribute): + founddisplay = founddisplay + 1 + if isinstance(attr, BrowsableAttribute): + foundbrowsable = foundbrowsable + 1 + SimpleClass.TestObject(obj) + found_display_on_property = 0 + for attr in Attribute.GetCustomAttributes(tp.GetProperty("X")): + if isinstance(attr, DebuggerDisplayAttribute): + found_display_on_property = found_display_on_property + 1 + found_display_on_method = 0 + for attr in Attribute.GetCustomAttributes(tp.GetMethod("Method1")): + if isinstance(attr, BrowsableAttribute): + found_display_on_method = found_display_on_method + 1 + assert founddisplay == 1 + assert found_display_on_property == 1 + assert found_display_on_method == 1 + assert foundbrowsable == 1 + assert obj.X == 1.0 + SimpleClass.TestObjectProperty(obj, "X", 10.0) +def test_class_with_advanced_attribute(): + import clr + @clr.attribute(TestAttribute(1, 2, z = "A", W = "B")) + class ClassWithAttributes2(ISimpleInterface, SimpleClass): + pass + c = ClassWithAttributes2() +def test_more_subclasses(): + import clr + class SubClass1(SimpleClass): + X = clr.property(Double, 1.0) + def __init__(self): + super().__init__() + self.Y = 10.0 + SimpleClass.Pause(); + + @clr.attribute(DebuggerDisplay("X")) + + class SubClass2(SubClass1): + __namespace__ = "TestModule" + def __init__(self): + SimpleClass.Pause(); + super().__init__() + def IncrementThing(self): + return 6; + obj = SimpleClass.InvokeCtor(SubClass2) + + obj2 = SubClass2() + tp = obj.GetType() + obj.X = 5.0 + assert obj.Y == 10.0 + assert obj2.Y == 10.0 + assert obj.Initialized == True + assert obj2.Initialized == True + SimpleClass.Test1(obj) + obj = None + SimpleClass.Test2() + + def test_construction_from_clr(): import clr From 014ef2e298845a777f6e6c7a4b40dae1fa61408b Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Thu, 5 May 2022 22:14:39 +0200 Subject: [PATCH 07/14] - Expanded the way attributes amy be added - Fixed search bug in AssocAttribute - Added testing --- src/runtime/Resources/clr.py | 51 +++++++++++++++++++++++-------- src/runtime/Types/ClassDerived.cs | 3 +- tests/test_subclass.py | 7 ++++- 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/src/runtime/Resources/clr.py b/src/runtime/Resources/clr.py index a5ff60370..0f412452d 100644 --- a/src/runtime/Resources/clr.py +++ b/src/runtime/Resources/clr.py @@ -49,8 +49,14 @@ def __set__(self, instance, value): if not self.fset: raise AttributeError("%s is read-only" % self.__name__) return self.fset.__get__(instance, None)(value) - def add_attribute(self, attribute): - self._clr_attributes_.append(attribute) + def add_attribute(self, *args, **kwargs): + lst = [] + if len(args) > 0: + if isinstance(args[0], tuple): + lst = args + else: + lst = [(args[0], args[1:], kwargs)] + self._clr_attributes_.extend(lst) return self class property(object): @@ -68,9 +74,16 @@ def __get__(self, instance, owner): return v def __set__(self, instance, value): self.values[instance] = value - def add_attribute(self, attribute): - self._clr_attributes_.append(attribute) + def add_attribute(self, *args, **kwargs): + lst = [] + if len(args) > 0: + if isinstance(args[0], tuple): + lst = args + else: + lst = [(args[0], args[1:], kwargs)] + self._clr_attributes_.extend(lst) return self + def __call__(self, type, default): self2 = self.__class__(self._clr_property_type_, type, default) self2._clr_attributes_ = self._clr_attributes_ @@ -118,22 +131,34 @@ def __call__(self, func): def __get__(self, instance, owner): return self.__func.__get__(instance, owner) - def clr_attribute(self, attribute): - self._clr_attributes_.append(attribute) + def add_attribute(self, *args, **kwargs): + lst = [] + if len(args) > 0: + if isinstance(args[0], tuple): + lst = args + else: + lst = [(args[0], args[1:], kwargs)] + self._clr_attributes_.extend(lst) return self class attribute(object): - def __init__(self, attr, *args, **kwargs): - self.attr = attr + def __init__(self, *args, **kwargs): + lst = [] + if len(args) > 0: + if isinstance(args[0], tuple): + lst = args + else: + lst = [(args[0], args[1:], kwargs)] import Python.Runtime #todo: ensure that attributes only are pushed when @ is used. - #import inspect - #Python.Runtime.PythonDerivedType.Test(inspect.stack()[1].code_context) + self.attr = lst + for item in lst: + Python.Runtime.PythonDerivedType.PushAttribute(item) - Python.Runtime.PythonDerivedType.PushAttribute(attr) def __call__(self, x): import Python.Runtime - if Python.Runtime.PythonDerivedType.AssocAttribute(self.attr, x): - pass + for item in self.attr: + if Python.Runtime.PythonDerivedType.AssocAttribute(item, x): + pass return x diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 6404b7460..0eaf70524 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -872,7 +872,8 @@ public static bool AssocAttribute(PyObject obj, PyObject func) var tp = new PyTuple(obj); for (int i = 0; i < attributesStack.Count; i++) { - if (Equals(tp, attributesStack[i])) + + if (tp.BorrowNullable()== attributesStack[i].BorrowNullable()) { attributesStack.RemoveAt(i); if (!methodAssoc.TryGetValue(func, out var lst)) diff --git a/tests/test_subclass.py b/tests/test_subclass.py index 828435183..e0fe5f647 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -12,7 +12,7 @@ from System.ComponentModel import (Browsable, BrowsableAttribute) import pytest from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, - FunctionsTest, GenericVirtualMethodTest, ISimpleInterface, SimpleClass, TestAttribute) + FunctionsTest, GenericVirtualMethodTest, ISimpleInterface, SimpleClass, TestAttribute, TestAttributeAttribute) from System.Collections.Generic import List @@ -339,7 +339,12 @@ def test_class_with_advanced_attribute(): @clr.attribute(TestAttribute(1, 2, z = "A", W = "B")) class ClassWithAttributes2(ISimpleInterface, SimpleClass): pass + @clr.attribute(TestAttributeAttribute, 1, 2, z = "A", W = "B") + class ClassWithAttributes3(ISimpleInterface, SimpleClass): + X = clr.property(Double, 1.0).add_attribute(TestAttributeAttribute, 1, 2) + c = ClassWithAttributes2() + c2 = ClassWithAttributes3() def test_more_subclasses(): import clr class SubClass1(SimpleClass): From 2449bd210b76268e2986d1fd04122fd06369bd4d Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Fri, 6 May 2022 08:30:13 +0200 Subject: [PATCH 08/14] added support for creating abstract classes using a __clr_abstract__ attribute. --- src/python_tests_runner/PythonTestRunner.cs | 1 + src/runtime/Types/ClassDerived.cs | 11 ++++++++++- tests/test_subclass.py | 10 +++++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/python_tests_runner/PythonTestRunner.cs b/src/python_tests_runner/PythonTestRunner.cs index b4f030fd7..58b113e33 100644 --- a/src/python_tests_runner/PythonTestRunner.cs +++ b/src/python_tests_runner/PythonTestRunner.cs @@ -42,6 +42,7 @@ static IEnumerable PythonTestCases() yield return new[] { "test_subclass", "test_class_with_attributes" }; yield return new[] { "test_subclass", "test_class_with_advanced_attribute" }; yield return new[] { "test_subclass", "test_more_subclasses" }; + yield return new[] { "test_subclass", "abstract_test" }; } /// diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 0eaf70524..a819c0e7d 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -280,8 +280,16 @@ internal static Type CreateDerivedType(string name, baseClass = typeof(object); } + bool isAbstract = false; + if (py_dict != null && Runtime.PyDict_Check(py_dict)) + { + using var dict = new PyDict(py_dict); + if (dict.HasKey("__clr_abstract__")) + isAbstract = true; + } + TypeBuilder typeBuilder = moduleBuilder.DefineType(name, - TypeAttributes.Public | TypeAttributes.Class, + TypeAttributes.Public | TypeAttributes.Class | (isAbstract ? TypeAttributes.Abstract : 0), baseClass, interfaces.ToArray()); @@ -359,6 +367,7 @@ internal static Type CreateDerivedType(string name, if (py_dict != null && Runtime.PyDict_Check(py_dict)) { using var dict = new PyDict(py_dict); + if (dict.HasKey("__clr_attributes__")) { var attributes = new PyList(dict["__clr_attributes__"]); diff --git a/tests/test_subclass.py b/tests/test_subclass.py index e0fe5f647..0aac33481 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -376,7 +376,15 @@ def IncrementThing(self): obj = None SimpleClass.Test2() - +def abstract_test(): + class abstractClass(SimpleClass): + __clr_abstract__ = True + failed = False + try: + abstractClass() + except: + failed = True + assert failed def test_construction_from_clr(): import clr From 7980fc4cf5cddab41856d181f57274568def9d3c Mon Sep 17 00:00:00 2001 From: Rolf Madsen Date: Tue, 24 May 2022 10:45:43 +0200 Subject: [PATCH 09/14] Improved AttributeError when looking for a symbol that does not exist inside a .NET module. --- src/python_tests_runner/PythonTestRunner.cs | 2 +- src/runtime/Types/ClassDerived.cs | 1 + src/runtime/Types/ModuleObject.cs | 2 +- src/testing/subclasstest.cs | 9 ++++++++- tests/test_subclass.py | 18 +++++++++++++++++- 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/python_tests_runner/PythonTestRunner.cs b/src/python_tests_runner/PythonTestRunner.cs index 58b113e33..9562c6a8d 100644 --- a/src/python_tests_runner/PythonTestRunner.cs +++ b/src/python_tests_runner/PythonTestRunner.cs @@ -18,7 +18,7 @@ public class PythonTestRunner public void SetUp() { Python.Runtime.Runtime.PythonDLL = - "/Library/Frameworks/Python.framework/Versions/3.10/lib/libpython3.10.dylib"; + "C:\\Python37.2\\python37.dll"; PythonEngine.Initialize(); } diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index a819c0e7d..726e4e41f 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -871,6 +871,7 @@ public class PythonDerivedType internal static Dictionary> methodAssoc = new Dictionary>(); public static void PushAttribute(PyObject obj) { + using var _ = Py.GIL(); var tp = new PyTuple(obj); attributesStack.Add(tp); } diff --git a/src/runtime/Types/ModuleObject.cs b/src/runtime/Types/ModuleObject.cs index af8c28ebd..73b8ff339 100644 --- a/src/runtime/Types/ModuleObject.cs +++ b/src/runtime/Types/ModuleObject.cs @@ -339,7 +339,7 @@ public static NewReference tp_getattro(BorrowedReference ob, BorrowedReference k if (attr.IsNull()) { - Exceptions.SetError(Exceptions.AttributeError, name); + Exceptions.SetError(Exceptions.AttributeError, $"name '{name}' is not defined in module '{self.moduleName}'."); return default; } diff --git a/src/testing/subclasstest.cs b/src/testing/subclasstest.cs index 989929ea3..cdfee3aed 100644 --- a/src/testing/subclasstest.cs +++ b/src/testing/subclasstest.cs @@ -130,7 +130,10 @@ public interface ISimpleInterface { bool Ok(); } - + public interface ISimpleInterface2 + { + int Execute(CancellationToken token); + } public class TestAttributeAttribute: Attribute { public int X { get; set; } @@ -168,6 +171,10 @@ public static void TestObject(object obj) if (!si.Ok()) throw new Exception(); + }else if (obj is ISimpleInterface2 si2) + { + si2.Execute(CancellationToken.None); + } else { diff --git a/tests/test_subclass.py b/tests/test_subclass.py index 0aac33481..e941d0691 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -10,9 +10,11 @@ from System import (Console, Attribute, Double) from System.Diagnostics import (DebuggerDisplay, DebuggerDisplayAttribute, Debug) from System.ComponentModel import (Browsable, BrowsableAttribute) +from System.Threading import (CancellationToken) import pytest from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, - FunctionsTest, GenericVirtualMethodTest, ISimpleInterface, SimpleClass, TestAttribute, TestAttributeAttribute) + FunctionsTest, GenericVirtualMethodTest, ISimpleInterface, SimpleClass, TestAttribute, TestAttributeAttribute, ISimpleInterface2) +import Python.Test from System.Collections.Generic import List @@ -290,12 +292,26 @@ def Ok(self): class DualSubClass2(ISimpleInterface): def Ok(self): return True + class DualSubClass3(ISimpleInterface2): + def Execute(self, cancellationToken): + return 0 + try: + class DualSubClass4(Python.Test.ISimpleInterface3): + def Execute(self, cancellationToken): + return 0 + assert False # An exception should be thrown. + except AttributeError as ae: + assert ("not defined" in str(ae)) obj = DualSubClass() SimpleClass.TestObject(obj) obj = DualSubClass2() SimpleClass.TestObject(obj) + obj2 = DualSubClass3(); + SimpleClass.TestObject(obj2) + #obj2.Execute(CancellationToken.None) + def test_class_with_attributes(): import clr @clr.attribute(Browsable(False)) From cf2c27a5903b5c7554a8657c1e0dd88e86e29aa6 Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Tue, 28 Jun 2022 11:55:28 +0200 Subject: [PATCH 10/14] Added test to detect an issue with object construction. --- src/testing/subclasstest.cs | 26 ++++++++++++++++++++++++++ tests/test_subclass.py | 23 ++++++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/testing/subclasstest.cs b/src/testing/subclasstest.cs index cdfee3aed..943366499 100644 --- a/src/testing/subclasstest.cs +++ b/src/testing/subclasstest.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.Threading; +using Python.Runtime; + namespace Python.Test { public interface IInterfaceTest @@ -215,6 +217,30 @@ public static object InvokeCtor(Type t) return obj; } + public object TestObj { get; set; } + + public static object TestOnType(Type t) + { + using (Py.GIL()) + { + var obj = (SimpleClass) Activator.CreateInstance(t); + //obj = obj.ToPython().As(); + obj.TestObj = new object(); + var py = obj.ToPython(); + var man = py.As(); + if (!ReferenceEquals(man, obj)) + throw new Exception("Same object expected"); + var setObj = py.GetAttr("TestObj").As(); + if (setObj == null) + throw new NullReferenceException(); + if (ReferenceEquals(setObj, obj.TestObj) == false) + throw new Exception("!!"); + + + return obj; + } + } + public static void Pause() { diff --git a/tests/test_subclass.py b/tests/test_subclass.py index e941d0691..585947d56 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -361,9 +361,27 @@ class ClassWithAttributes3(ISimpleInterface, SimpleClass): c = ClassWithAttributes2() c2 = ClassWithAttributes3() + +def test_subclass_ctor(): + import clr + class SubClass0(SimpleClass): + pass + class SubClass1(SubClass0): + def __init__(self): + super().__init__() + class SubClass2(SubClass1): + __namespace__ = "TestModule" + def __init__(self): + super().__init__() + SimpleClass.TestOnType(SubClass0) + SimpleClass.TestOnType(SubClass1) + SimpleClass.TestOnType(SubClass2) + def test_more_subclasses(): import clr - class SubClass1(SimpleClass): + class SubClass0(SimpleClass): + pass + class SubClass1(SubClass0): X = clr.property(Double, 1.0) def __init__(self): super().__init__() @@ -379,6 +397,9 @@ def __init__(self): super().__init__() def IncrementThing(self): return 6; + SimpleClass.TestOnType(SubClass0) + SimpleClass.TestOnType(SubClass1) + SimpleClass.TestOnType(SubClass2) obj = SimpleClass.InvokeCtor(SubClass2) obj2 = SubClass2() From adc6613f2f9410fbdfbd75748f523892751de6d4 Mon Sep 17 00:00:00 2001 From: Rolf Madsen Date: Fri, 2 Sep 2022 09:38:42 +0200 Subject: [PATCH 11/14] Added support for marking properties with python types. Added support for specifying attributes on property-methods (eg. get/set attributes). --- src/runtime/PythonTypes/PyObject.cs | 8 +-- .../PythonTypes/PythonTypeAttribute.cs | 27 ++++++++++ src/runtime/Resources/clr.py | 12 +++-- src/runtime/Types/ClassDerived.cs | 52 ++++++++++++++++--- 4 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 src/runtime/PythonTypes/PythonTypeAttribute.cs diff --git a/src/runtime/PythonTypes/PyObject.cs b/src/runtime/PythonTypes/PyObject.cs index bda2d9c02..31e00a17b 100644 --- a/src/runtime/PythonTypes/PyObject.cs +++ b/src/runtime/PythonTypes/PyObject.cs @@ -25,7 +25,7 @@ public partial class PyObject : DynamicObject, IDisposable, ISerializable /// Trace stack for PyObject's construction /// public StackTrace Traceback { get; } = new StackTrace(1); -#endif +#endif protected IntPtr rawPtr = IntPtr.Zero; internal readonly int run = Runtime.GetRun(); @@ -165,7 +165,7 @@ public static PyObject FromManagedObject(object ob) { if (!Converter.ToManaged(obj, t, out var result, true)) { - throw new InvalidCastException("cannot convert object to target type", + throw new InvalidCastException($"Cannot convert object to target type '{t}'.", PythonException.FetchCurrentOrNull(out _)); } return result; @@ -235,7 +235,7 @@ public void Dispose() { GC.SuppressFinalize(this); Dispose(true); - + } internal StolenReference Steal() @@ -1325,7 +1325,7 @@ private bool TryCompare(PyObject arg, int op, out object @out) } return true; } - + public override bool TryBinaryOperation(BinaryOperationBinder binder, object arg, out object? result) { using var _ = Py.GIL(); diff --git a/src/runtime/PythonTypes/PythonTypeAttribute.cs b/src/runtime/PythonTypes/PythonTypeAttribute.cs new file mode 100644 index 000000000..7dbc413a9 --- /dev/null +++ b/src/runtime/PythonTypes/PythonTypeAttribute.cs @@ -0,0 +1,27 @@ +using System; + +namespace Python.Runtime; + +/// +/// Marks a property type with a specific python type. Normally, properties has .NET types, but if the property has a python type, +/// that cannot be represented in the propert type info, so this attribute is used to mark the property with the corresponding python type. +/// +public class PythonTypeAttribute : Attribute +{ + /// Type name. + public string TypeName { get; } + + /// Importable module name. + public string Module { get; } + + /// + /// Creates a new instance of PythonTypeAttribute. + /// + /// + /// + public PythonTypeAttribute(string pyTypeModule, string pyTypeName) + { + TypeName = pyTypeName; + Module = pyTypeModule; + } +} diff --git a/src/runtime/Resources/clr.py b/src/runtime/Resources/clr.py index 0f412452d..9d50d967a 100644 --- a/src/runtime/Resources/clr.py +++ b/src/runtime/Resources/clr.py @@ -61,7 +61,7 @@ def add_attribute(self, *args, **kwargs): class property(object): - def __init__(self, type, default): + def __init__(self, type, default = None): import weakref self._clr_property_type_ = type self.default = default @@ -70,9 +70,14 @@ def __init__(self, type, default): self.fget = 1 self.fset = 1 def __get__(self, instance, owner): + if self.fget != 1: + return self.fget(instance) v = self.values.get(instance, self.default) return v def __set__(self, instance, value): + if self.fset != 1: + self.fset(instance,value) + return self.values[instance] = value def add_attribute(self, *args, **kwargs): lst = [] @@ -84,8 +89,9 @@ def add_attribute(self, *args, **kwargs): self._clr_attributes_.extend(lst) return self - def __call__(self, type, default): - self2 = self.__class__(self._clr_property_type_, type, default) + def __call__(self, func): + self2 = self.__class__(self._clr_property_type_, None) + self2.fget = func self2._clr_attributes_ = self._clr_attributes_ return self2 class clrmethod(object): diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 726e4e41f..ec7527925 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -145,6 +145,10 @@ static CustomAttributeBuilder AddAttribute(PyObject attr) // named arguments and named properties or fields the same way. var tp = new PyTuple(attr); Type attribute = (Type) tp[0].AsManagedObject(typeof(object)); + if (typeof(Attribute).IsAssignableFrom(attribute) == false) + { + throw new Exception("This type is not an attribute type."); + } var args = (PyObject[]) tp[1].AsManagedObject(typeof(PyObject[])); var dict = new PyDict(tp[2]); @@ -168,9 +172,10 @@ static CustomAttributeBuilder AddAttribute(PyObject attr) if (parameter.ParameterType.IsArray && null != parameter.GetCustomAttribute(typeof(ParamArrayAttribute))) { int cnt = args.Length - i; - var values = Array.CreateInstance(parameter.ParameterType.GetElementType(), cnt); + var elemType = parameter.ParameterType.GetElementType(); + var values = Array.CreateInstance(elemType, cnt); for(int j = 0; j < cnt; j++) - values.SetValue(args[i + j], j); + values.SetValue(args[i + j].AsManagedObject(typeof(object)), j); paramValues.Add(values); break; } @@ -750,28 +755,61 @@ private static void AddPythonProperty(string propertyName, PyObject func, TypeBu MethodAttributes.SpecialName; using var pyPropertyType = func.GetAttr("_clr_property_type_"); - var propertyType = pyPropertyType.AsManagedObject(typeof(Type)) as Type; + var pyNativeType = new PyType(pyPropertyType); + Converter.ToManaged(pyPropertyType, typeof(Type), out var result, false); + var propertyType = result as Type; + string pyTypeName = null; + string pyTypeModule = null; if (propertyType == null) { - throw new ArgumentException("_clr_property_type must be a CLR type"); + propertyType = typeof(PyObject); + pyTypeModule = pyNativeType.GetAttr("__module__").ToString(); + //throw new ArgumentException("_clr_property_type must be a CLR type"); + pyTypeName = pyNativeType.Name; } PropertyBuilder propertyBuilder = typeBuilder.DefineProperty(propertyName, PropertyAttributes.None, propertyType, null); - if (func.HasAttr("_clr_attributes_")) + if (func.HasAttr("_clr_attributes_")) + { + using (var attributes = new PyList(func.GetAttr("_clr_attributes_"))) + { + foreach (var attr in attributes) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + propertyBuilder.SetCustomAttribute(builder); + } + } + } + + if (pyTypeName != null) + { + var cb = new CustomAttributeBuilder(typeof(PythonTypeAttribute).GetConstructors().First(), + new object[] {pyTypeModule, pyTypeName}); + propertyBuilder.SetCustomAttribute(cb); + } + + { + // load attribute set on functions + foreach (var fname in new[]{ "fget", "fset" }) { - using (var attributes = new PyList(func.GetAttr("_clr_attributes_"))) + using var pyfget = func.GetAttr(fname); + if (methodAssoc.TryGetValue(pyfget, out var lst)) { - foreach (var attr in attributes) + foreach (var attr in lst) { var builder = AddAttribute(attr); if (builder == null) throw new Exception(); propertyBuilder.SetCustomAttribute(builder); } + + methodAssoc.Remove(func); } } + } if (func.HasAttr("fget")) { From 9373a03a9938a68f8b447d2781b0ddf77daac83d Mon Sep 17 00:00:00 2001 From: Rolf Madsen Date: Fri, 2 Sep 2022 15:03:48 +0200 Subject: [PATCH 12/14] Fixed issue calling base-base class implementation of methods. --- src/python_tests_runner/PythonTestRunner.cs | 1 + src/runtime/Types/ClassDerived.cs | 33 +++++++++++---------- src/testing/subclasstest.cs | 18 ++++++++--- tests/test_subclass.py | 24 +++++++++++++++ 4 files changed, 57 insertions(+), 19 deletions(-) diff --git a/src/python_tests_runner/PythonTestRunner.cs b/src/python_tests_runner/PythonTestRunner.cs index 9562c6a8d..dcc76e9d9 100644 --- a/src/python_tests_runner/PythonTestRunner.cs +++ b/src/python_tests_runner/PythonTestRunner.cs @@ -42,6 +42,7 @@ static IEnumerable PythonTestCases() yield return new[] { "test_subclass", "test_class_with_attributes" }; yield return new[] { "test_subclass", "test_class_with_advanced_attribute" }; yield return new[] { "test_subclass", "test_more_subclasses" }; + yield return new[] { "test_subclass", "test_more_subclasses2" }; yield return new[] { "test_subclass", "abstract_test" }; } diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index ec7527925..1465bc035 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -522,23 +522,26 @@ private static void AddVirtualMethod(MethodInfo method, Type baseType, TypeBuild string? baseMethodName = null; if (!method.IsAbstract) { - baseMethodName = "_" + baseType.Name + "__" + method.Name; - MethodBuilder baseMethodBuilder = typeBuilder.DefineMethod(baseMethodName, - MethodAttributes.Public | - MethodAttributes.Final | - MethodAttributes.HideBySig, - method.ReturnType, - parameterTypes); - - // emit the assembly for calling the original method using call instead of callvirt - ILGenerator baseIl = baseMethodBuilder.GetILGenerator(); - baseIl.Emit(OpCodes.Ldarg_0); - for (var i = 0; i < parameters.Length; ++i) + baseMethodName = "_" + method.DeclaringType.Name + "__" + method.Name; + if (baseType.GetMethod(baseMethodName) == null) { - baseIl.Emit(OpCodes.Ldarg, i + 1); + MethodBuilder baseMethodBuilder = typeBuilder.DefineMethod(baseMethodName, + MethodAttributes.Public | + MethodAttributes.Final | + MethodAttributes.HideBySig, + method.ReturnType, + parameterTypes); + + // emit the assembly for calling the original method using call instead of callvirt + ILGenerator baseIl = baseMethodBuilder.GetILGenerator(); + baseIl.Emit(OpCodes.Ldarg_0); + for (var i = 0; i < parameters.Length; ++i) + { + baseIl.Emit(OpCodes.Ldarg, i + 1); + } + baseIl.Emit(OpCodes.Call, method); + baseIl.Emit(OpCodes.Ret); } - baseIl.Emit(OpCodes.Call, method); - baseIl.Emit(OpCodes.Ret); } // override the original method with a new one that dispatches to python diff --git a/src/testing/subclasstest.cs b/src/testing/subclasstest.cs index 943366499..9b6071c15 100644 --- a/src/testing/subclasstest.cs +++ b/src/testing/subclasstest.cs @@ -151,8 +151,17 @@ public TestAttributeAttribute(int x, int y, string z = "x") } } + public abstract class SimpleClassBase + { + private int counter; + public virtual int IncrementThing() + { + return counter++; + } - public class SimpleClass + } + + public abstract class SimpleClass : SimpleClassBase { public bool Initialized; @@ -160,10 +169,11 @@ public SimpleClass() { Initialized = true; } - private int counter = 0; - public virtual int IncrementThing() + + public int CallIncrementThing() { - return ++counter; + var x = IncrementThing(); + return x; } public static void TestObject(object obj) diff --git a/tests/test_subclass.py b/tests/test_subclass.py index 585947d56..cdce67786 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -396,6 +396,7 @@ def __init__(self): SimpleClass.Pause(); super().__init__() def IncrementThing(self): + super().IncrementThing() return 6; SimpleClass.TestOnType(SubClass0) SimpleClass.TestOnType(SubClass1) @@ -465,3 +466,26 @@ class Derived(BaseClass): import gc gc.collect() +def test_more_subclasses2(): + import clr + class SubClass50(SimpleClass): + def __init__(self): + super().__init__() + def IncrementThing(self): + return super().IncrementThing() + + @clr.attribute(DebuggerDisplay("X")) + + class SubClass51(SubClass50): + __namespace__ = "TestModule" + def __init__(self): + super().__init__() + + def IncrementThing(self): + return super().IncrementThing() + 10 + x = SubClass51() + print(x.CallIncrementThing()) + print(x.CallIncrementThing()) + print(x.CallIncrementThing()) + + From ab124814aa6b3b305afa2b7c11412556e513d5c7 Mon Sep 17 00:00:00 2001 From: Rolf Madsen Date: Fri, 28 Oct 2022 14:03:25 +0200 Subject: [PATCH 13/14] Merged with Pythonnet 3.0 RC.6. --- src/runtime/TypeManager.cs | 16 +++++----------- src/runtime/Types/ClassDerived.cs | 6 +++--- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 1099d6430..9746bc0ce 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -416,17 +416,11 @@ internal static NewReference CreateSubType(BorrowedReference py_name, IEnumerabl // create the new managed type subclassing the base managed type var baseClass = py_base_type.FirstOrDefault(); - if (null == baseClass) - { - return ReflectedClrType.CreateSubclass(baseClass, interfaces, name, - ns: (string?)namespaceStr, - assembly: (string?)assembly, - dict: dictRef); - } - else - { - return Exceptions.RaiseTypeError("invalid base class, expected CLR class type"); - } + + return ReflectedClrType.CreateSubclass(baseClass, interfaces, name, + ns: (string?)namespaceStr, + assembly: (string?)assembly, + dict: dictRef); } internal static IntPtr WriteMethodDef(IntPtr mdef, IntPtr name, IntPtr func, PyMethodFlags flags, IntPtr doc) diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 1465bc035..e06053c97 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -625,8 +625,8 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde } using var pyReturnType = func.GetAttr("_clr_return_type_"); - usingvar attributes = new PyList(func.GetAttr("_clr_attributes_"))) - using (var pyArgTypes = func.GetAttr("_clr_arg_types_"); + using var attributes = new PyList(func.GetAttr("_clr_attributes_")); + using var pyArgTypes = func.GetAttr("_clr_arg_types_"); using var pyArgTypesIter = PyIter.GetIter(pyArgTypes); var returnType = pyReturnType.AsManagedObject(typeof(Type)) as Type; if (returnType == null) @@ -1160,7 +1160,7 @@ public static void InvokeCtor(IPythonDerivedType obj, string origCtorName, objec // This doesn't actually destroy the object, it just sets the reference to this object // to be a weak reference and it will be destroyed when the C# object is destroyed. - PyType cc = ClassManager.GetClass(obj.GetType()); + PyType cc = ReflectedClrType.GetOrCreate(obj.GetType()); var args2 = new PyObject[args.Length]; for (int i = 0; i < args.Length; i++) args2[i] = args[i].ToPython(); From bb988cd523278114370d7a36ed50fb3572f9c4c0 Mon Sep 17 00:00:00 2001 From: Rolf Madsen Date: Fri, 18 Nov 2022 13:50:08 +0100 Subject: [PATCH 14/14] Cleanup and comments Added a few cleanups and comments. --- src/runtime/Resources/clr.py | 29 +++++++++++++ src/runtime/Types/ClassDerived.cs | 71 +++++++++++++++++++++---------- src/runtime/Types/MetaType.cs | 24 +++-------- src/runtime/Types/ModuleObject.cs | 4 +- 4 files changed, 87 insertions(+), 41 deletions(-) diff --git a/src/runtime/Resources/clr.py b/src/runtime/Resources/clr.py index 9d50d967a..7bda24880 100644 --- a/src/runtime/Resources/clr.py +++ b/src/runtime/Resources/clr.py @@ -49,7 +49,12 @@ def __set__(self, instance, value): if not self.fset: raise AttributeError("%s is read-only" % self.__name__) return self.fset.__get__(instance, None)(value) + + # TODO: I am not sure this add_attribute is actually necessary. def add_attribute(self, *args, **kwargs): + """Adds an attribute to this class. + If the first argument is a tuple we assume it is a tuple containing everything to initialize the attribute. + Otherwise, the first arg should be a .net type implementing Attribute.""" lst = [] if len(args) > 0: if isinstance(args[0], tuple): @@ -60,7 +65,17 @@ def add_attribute(self, *args, **kwargs): return self class property(object): + """ + property constructor for creating properties with implicit get/set. + + It can be used as such: + e.g.:: + + class X(object): + A = clr.property(Double, 3.14)\ + .add_attribute(Browsable(False)) + """ def __init__(self, type, default = None): import weakref self._clr_property_type_ = type @@ -79,7 +94,11 @@ def __set__(self, instance, value): self.fset(instance,value) return self.values[instance] = value + def add_attribute(self, *args, **kwargs): + """Adds an attribute to this class. + If the first argument is a tuple we assume it is a tuple containing everything to initialize the attribute. + Otherwise, the first arg should be a .net type implementing Attribute.""" lst = [] if len(args) > 0: if isinstance(args[0], tuple): @@ -138,6 +157,9 @@ def __get__(self, instance, owner): return self.__func.__get__(instance, owner) def add_attribute(self, *args, **kwargs): + """Adds an attribute to this class. + If the first argument is a tuple we assume it is a tuple containing everything to initialize the attribute. + Otherwise, the first arg should be a .net type implementing Attribute.""" lst = [] if len(args) > 0: if isinstance(args[0], tuple): @@ -148,7 +170,14 @@ def add_attribute(self, *args, **kwargs): return self class attribute(object): + """ + Class decorator for adding attributes to .net python classes. + e.g.:: + @attribute(DisplayName("X Class")) + class X(object): + pass + """ def __init__(self, *args, **kwargs): lst = [] if len(args) > 0: diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index bd04ffe94..b5bdad47e 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -276,7 +276,10 @@ internal static Type CreateDerivedType(string name, Type baseClass = baseType; var interfaces = new HashSet { typeof(IPythonDerivedType) }; - foreach(var t in interfaces2) interfaces.Add(t); + + foreach(var interfaceType in interfaces2) + interfaces.Add(interfaceType); + // if the base type is an interface then use System.Object as the base class // and add the base type to the list of interfaces this new class will implement. if (baseType.IsInterface) @@ -285,6 +288,7 @@ internal static Type CreateDerivedType(string name, baseClass = typeof(object); } + // __clr_abstract__ is used to create an abstract class. bool isAbstract = false; if (py_dict != null && Runtime.PyDict_Check(py_dict)) { @@ -298,10 +302,11 @@ internal static Type CreateDerivedType(string name, baseClass, interfaces.ToArray()); - // add a field for storing the python object pointer - // FIXME: fb not used + // add a field for storing the python object pointer, + // but only if the baseclass does not already have it. if (baseClass.GetField(PyObjName, PyObjFlags) == null) { + // FIXME: fb not used FieldBuilder fb = typeBuilder.DefineField(PyObjName, #pragma warning disable CS0618 // Type or member is obsolete. OK for internal use. typeof(UnsafeReferenceWithRun), @@ -373,9 +378,10 @@ internal static Type CreateDerivedType(string name, { using var dict = new PyDict(py_dict); + // if there are any attributes, add them. if (dict.HasKey("__clr_attributes__")) { - var attributes = new PyList(dict["__clr_attributes__"]); + using var attributes = new PyList(dict["__clr_attributes__"]); foreach (var attr in attributes) { var builder = AddAttribute(attr); @@ -442,12 +448,10 @@ internal static Type CreateDerivedType(string name, return type; } - private static Dictionary pyTypeLookup = new Dictionary(); - /// /// Add a constructor override that calls the python ctor after calling the base type constructor. /// - /// constructor to be called before calling the python ctor + /// constructor to be called before calling the python ctor. This can be null if there is no constructor. /// Python callable object /// TypeBuilder for the new type the ctor is to be added to private static void AddConstructor(ConstructorInfo ctor, Type baseType, TypeBuilder typeBuilder) @@ -665,7 +669,7 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde methodBuilder.SetCustomAttribute(builder); } - if (methodAssoc.TryGetValue(func, out var lst)) + if (MethodAttributeAssociations.TryGetValue(func, out var lst)) { foreach(var attr in lst) { @@ -674,7 +678,7 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde methodBuilder.SetCustomAttribute(builder); } - methodAssoc.Remove(func); + MethodAttributeAssociations.Remove(func); } @@ -770,11 +774,12 @@ private static void AddPythonProperty(string propertyName, PyObject func, TypeBu var propertyType = result as Type; string pyTypeName = null; string pyTypeModule = null; + // if the property type is null, we assume that it is a python type + // and not a C# type, in this case the property is just a PyObject type instead. if (propertyType == null) { propertyType = typeof(PyObject); pyTypeModule = pyNativeType.GetAttr("__module__").ToString(); - //throw new ArgumentException("_clr_property_type must be a CLR type"); pyTypeName = pyNativeType.Name; } @@ -807,7 +812,7 @@ private static void AddPythonProperty(string propertyName, PyObject func, TypeBu foreach (var fname in new[]{ "fget", "fset" }) { using var pyfget = func.GetAttr(fname); - if (methodAssoc.TryGetValue(pyfget, out var lst)) + if (MethodAttributeAssociations.TryGetValue(pyfget, out var lst)) { foreach (var attr in lst) { @@ -816,7 +821,7 @@ private static void AddPythonProperty(string propertyName, PyObject func, TypeBu propertyBuilder.SetCustomAttribute(builder); } - methodAssoc.Remove(func); + MethodAttributeAssociations.Remove(func); } } } @@ -915,28 +920,45 @@ private static ModuleBuilder GetModuleBuilder(string assemblyName, string module [Obsolete(Util.InternalUseOnly)] public class PythonDerivedType { - private static List attributesStack = new List(); - internal static Dictionary> methodAssoc = new Dictionary>(); - public static void PushAttribute(PyObject obj) + /// Tracks the attributes pushed with PushAttributes. + static List attributesStack = new (); + /// + /// This field track associations between python functions and associated attributes. + /// + internal static Dictionary> MethodAttributeAssociations = new (); + + /// + /// This pushes an attribute on the current attribute stack. + /// This happens when invoking e.g `@(attribute(Browsable(False))`. + /// + /// The pushed attribute. + /// This is should not be an attribute instance, but a tuple from which it can be created. + public static void PushAttribute(PyObject attribute) { using var _ = Py.GIL(); - var tp = new PyTuple(obj); + var tp = new PyTuple(attribute); attributesStack.Add(tp); } - public static bool AssocAttribute(PyObject obj, PyObject func) + /// + /// Associates an attribute with a function. + /// + /// + /// + /// + + public static bool AssocAttribute(PyObject attribute, PyObject func) { using var _ = Py.GIL(); - var tp = new PyTuple(obj); + var tp = new PyTuple(attribute); for (int i = 0; i < attributesStack.Count; i++) { - - if (tp.BorrowNullable()== attributesStack[i].BorrowNullable()) + if (tp.BorrowNullable() == attributesStack[i].BorrowNullable()) { attributesStack.RemoveAt(i); - if (!methodAssoc.TryGetValue(func, out var lst)) + if (!MethodAttributeAssociations.TryGetValue(func, out var lst)) { - lst = methodAssoc[func] = new List(); + lst = MethodAttributeAssociations[func] = new List(); } lst.Add(tp); return true; @@ -947,6 +969,10 @@ public static bool AssocAttribute(PyObject obj, PyObject func) } + /// + /// Pops the current attribute stack and returns it. Any future pushed attributes will be in a new list. + /// + /// public static IEnumerable PopAttributes() { if (attributesStack.Count == 0) return Array.Empty(); @@ -954,6 +980,7 @@ public static IEnumerable PopAttributes() attributesStack = new List(); return attrs; } + internal const string PyObjName = "__pyobj__"; internal const BindingFlags PyObjFlags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.FlattenHierarchy; diff --git a/src/runtime/Types/MetaType.cs b/src/runtime/Types/MetaType.cs index 459796ebf..d198ddd41 100644 --- a/src/runtime/Types/MetaType.cs +++ b/src/runtime/Types/MetaType.cs @@ -82,12 +82,7 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, BorrowedReference bases = Runtime.PyTuple_GetItem(args, 1); BorrowedReference dict = Runtime.PyTuple_GetItem(args, 2); - // We do not support multiple inheritance, so the bases argument - // should be a 1-item tuple containing the type we are subtyping. - // That type must itself have a managed implementation. We check - // that by making sure its metatype is the CLR metatype. - - + // Extract interface types and base class types. List interfaces = new List(); List baseType = new List(); @@ -105,25 +100,18 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, } } + // if the base type count is 0, there might still be interfaces to implement. if (baseType.Count == 0) { baseType.Add(new ClassBase(typeof(object))); } - + // Multiple inheritance is not supported, unless the other types are interfaces if (baseType.Count > 1) { return Exceptions.RaiseTypeError("cannot use multiple inheritance with managed classes"); } - /* - BorrowedReference mt = Runtime.PyObject_TYPE(baseType); - - if (!(mt == PyCLRMetaType || mt == Runtime.PyTypeType)) - { - return Exceptions.RaiseTypeError("invalid metatype"); - }*/ - // Ensure that the reflected type is appropriate for subclassing, // disallowing subclassing of delegates, enums and array types. @@ -146,7 +134,8 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, return Exceptions.RaiseTypeError("subclasses of managed classes do not support __slots__"); } - // If __assembly__ or __namespace__ are in the class dictionary then create + // If the base class has a parameterless constructor, or + // if __assembly__ or __namespace__ are in the class dictionary then create // a managed sub type. // This creates a new managed type that can be used from .net to call back // into python. @@ -156,7 +145,7 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, var ctor = btt?.GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) .FirstOrDefault(x => x.GetParameters().Any() == false); using var clsDict = new PyDict(dict); - + if (clsDict.HasKey("__assembly__") || clsDict.HasKey("__namespace__") || (ctor != null)) { @@ -167,7 +156,6 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, } return TypeManager.CreateSubType(name, baseType, interfaces, clsDict); } - } var base_type = Runtime.PyTuple_GetItem(bases, 0); diff --git a/src/runtime/Types/ModuleObject.cs b/src/runtime/Types/ModuleObject.cs index 73b8ff339..f3f9a0e67 100644 --- a/src/runtime/Types/ModuleObject.cs +++ b/src/runtime/Types/ModuleObject.cs @@ -132,8 +132,10 @@ public NewReference GetAttribute(string name, bool guess) return new NewReference(c); } - // attribute names + // Simplified imported attribute names // for attributes without the Attribute suffix, create an attribute builder. + // This means that imported attributes can be invoked using just e.g 'Browsable(False)' + // and not BrowsableAttribute(False) although its still an option. var qname2 = qname + "Attribute"; var type2 = AssemblyManager.LookupTypes(qname2).FirstOrDefault(t => t.IsPublic); if (type2 != null)