From e8ea7a6b2f0e37390033efdb5c8302597902d93d Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Wed, 27 Apr 2022 22:47:25 +0200 Subject: [PATCH 01/17] 1776 Inherit Generic Virtual Method Bug: Unit Test Added a unit test which consists ofa python class inheriting from a C# class which has a virtual generic method. In this version, this causes a InvalidProgramException to be thrown during class creation. --- src/testing/generictest.cs | 8 ++++++++ tests/test_subclass.py | 8 +++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/testing/generictest.cs b/src/testing/generictest.cs index 238435811..b333910c2 100644 --- a/src/testing/generictest.cs +++ b/src/testing/generictest.cs @@ -136,4 +136,12 @@ public static T[] EchoRange(T[] items) return items; } } + + public abstract class GenericVirtualMethodTest + { + public virtual Q VirtMethod(Q arg1) + { + return arg1; + } + } } diff --git a/tests/test_subclass.py b/tests/test_subclass.py index fa82c3663..3d4c1d40d 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -9,7 +9,7 @@ import System import pytest from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, - FunctionsTest) + FunctionsTest, GenericVirtualMethodTest) from System.Collections.Generic import List @@ -264,6 +264,12 @@ class TestX(System.Object): t = TestX() assert t.q == 1 +def test_virtual_generic_method(): + class OverloadingSubclass(GenericVirtualMethodTest): + __namespace__ = "test_virtual_generic_method_cls" + obj = OverloadingSubclass() + assert obj.VirtMethod[int](5) == 5 + def test_construction_from_clr(): import clr calls = [] From 691f207c0f2fe484624b90477764f073030afe97 Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Thu, 28 Apr 2022 08:48:08 +0200 Subject: [PATCH 02/17] 1776 Inherit Generic Virtual Method Bug: Fix Previously an exception was thrown during class creation if the python class inherited from a class with a virtual generic method. This has been fixed by ignoring generic methods when creating overloads in the generated class. Note, this means that generic virtual methods cannot be overridden in python code. --- src/runtime/Types/ClassDerived.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 02288faee..07367a04a 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -219,8 +219,9 @@ internal static Type CreateDerivedType(string name, var virtualMethods = new HashSet(); foreach (MethodInfo method in methods) { - if (!method.Attributes.HasFlag(MethodAttributes.Virtual) | - method.Attributes.HasFlag(MethodAttributes.Final)) + if (!method.Attributes.HasFlag(MethodAttributes.Virtual) + || method.Attributes.HasFlag(MethodAttributes.Final) + || method.IsGenericMethod) { continue; } From e27ad8c0ff5ab304b420341fa1a6908c7ecc72f0 Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Fri, 29 Apr 2022 08:14:26 +0200 Subject: [PATCH 03/17] 1774-ClassWithoutnamespace - Changed the behavior so that a .NET is always created when the base type is also a .NET type. --- src/runtime/Types/MetaType.cs | 10 +++++++++- tests/test_subclass.py | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/runtime/Types/MetaType.cs b/src/runtime/Types/MetaType.cs index 5b59f5139..db1a3ff2e 100644 --- a/src/runtime/Types/MetaType.cs +++ b/src/runtime/Types/MetaType.cs @@ -128,10 +128,18 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, if (null != dict) { using var clsDict = new PyDict(dict); - if (clsDict.HasKey("__assembly__") || clsDict.HasKey("__namespace__")) + + if (clsDict.HasKey("__assembly__") || clsDict.HasKey("__namespace__") + || (cb.type.Valid && cb.type.Value != null && cb.type.Value.GetConstructor(Array.Empty()) != null)) { + if (!clsDict.HasKey("__namespace__")) + { + clsDict["__namespace__"] = + (clsDict["__module__"].ToString()).ToPython(); + } return TypeManager.CreateSubType(name, base_type, clsDict); } + } // otherwise just create a basic type without reflecting back into the managed side. diff --git a/tests/test_subclass.py b/tests/test_subclass.py index 3d4c1d40d..8b15d31ff 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -267,6 +267,8 @@ class TestX(System.Object): def test_virtual_generic_method(): class OverloadingSubclass(GenericVirtualMethodTest): __namespace__ = "test_virtual_generic_method_cls" + class OverloadingSubclass2(OverloadingSubclass): + pass obj = OverloadingSubclass() assert obj.VirtMethod[int](5) == 5 From f1fa6968da3381543e3906a9cd01d4151a25b5b9 Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Tue, 3 May 2022 18:52:33 +0200 Subject: [PATCH 04/17] better support for multiple inheritance --- src/python_tests_runner/PythonTestRunner.cs | 4 ++ src/runtime/Runtime.cs | 6 ++ src/runtime/StateSerialization/MaybeType.cs | 4 +- src/runtime/TypeManager.cs | 7 ++- src/runtime/Types/ClassDerived.cs | 7 ++- src/runtime/Types/MetaType.cs | 61 +++++++++++++++------ src/runtime/Types/ReflectedClrType.cs | 3 +- src/testing/generictest.cs | 10 ++-- src/testing/subclasstest.cs | 21 +++++++ tests/test_subclass.py | 24 +++++++- 10 files changed, 117 insertions(+), 30 deletions(-) diff --git a/src/python_tests_runner/PythonTestRunner.cs b/src/python_tests_runner/PythonTestRunner.cs index 05298997b..98b2fbe68 100644 --- a/src/python_tests_runner/PythonTestRunner.cs +++ b/src/python_tests_runner/PythonTestRunner.cs @@ -17,6 +17,8 @@ public class PythonTestRunner [OneTimeSetUp] public void SetUp() { + Python.Runtime.Runtime.PythonDLL = + "/Library/Frameworks/Python.framework/Versions/3.10/lib/libpython3.10.dylib"; PythonEngine.Initialize(); } @@ -35,6 +37,8 @@ static IEnumerable PythonTestCases() // Add the test that you want to debug here. yield return new[] { "test_indexer", "test_boolean_indexer" }; yield return new[] { "test_delegate", "test_bool_delegate" }; + yield return new[] { "test_subclass", "test_virtual_generic_method" }; + yield return new[] { "test_subclass", "test_interface_and_class_impl2" }; } /// diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 6238119ff..79578b7c9 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -1824,6 +1824,12 @@ internal static void SetNoSiteFlag() return *Delegates.Py_NoSiteFlag; }); } + + internal static uint PyTuple_GetSize(BorrowedReference tuple) + { + IntPtr r = Delegates.PyTuple_Size(tuple); + return (uint)r.ToInt32(); + } } internal class BadPythonDllException : MissingMethodException diff --git a/src/runtime/StateSerialization/MaybeType.cs b/src/runtime/StateSerialization/MaybeType.cs index f3c96e369..549c5f29e 100644 --- a/src/runtime/StateSerialization/MaybeType.cs +++ b/src/runtime/StateSerialization/MaybeType.cs @@ -15,7 +15,6 @@ internal struct MaybeType : ISerializable const string SerializationName = "n"; readonly string name; readonly Type type; - public string DeletedMessage { get @@ -38,6 +37,7 @@ public Type Value public string Name => name; public bool Valid => type != null; + public Type ValueOrNull => type; public override string ToString() { @@ -61,4 +61,4 @@ public void GetObjectData(SerializationInfo serializationInfo, StreamingContext serializationInfo.AddValue(SerializationName, name); } } -} \ No newline at end of file +} diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 217b4820e..1099d6430 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -374,7 +374,7 @@ static PyTuple GetBaseTypeTuple(Type clrType) return new PyTuple(bases); } - internal static NewReference CreateSubType(BorrowedReference py_name, BorrowedReference py_base_type, BorrowedReference dictRef) + internal static NewReference CreateSubType(BorrowedReference py_name, IEnumerable py_base_type, IEnumerable interfaces, BorrowedReference dictRef) { // Utility to create a subtype of a managed type with the ability for the // a python subtype able to override the managed implementation @@ -415,9 +415,10 @@ internal static NewReference CreateSubType(BorrowedReference py_name, BorrowedRe } // create the new managed type subclassing the base managed type - if (ManagedType.GetManagedObject(py_base_type) is ClassBase baseClass) + var baseClass = py_base_type.FirstOrDefault(); + if (null == baseClass) { - return ReflectedClrType.CreateSubclass(baseClass, name, + return ReflectedClrType.CreateSubclass(baseClass, interfaces, name, ns: (string?)namespaceStr, assembly: (string?)assembly, dict: dictRef); diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 07367a04a..a45beb78f 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -144,6 +144,7 @@ internal static NewReference ToPython(IPythonDerivedType obj) /// internal static Type CreateDerivedType(string name, Type baseType, + IEnumerable interfaces2, BorrowedReference py_dict, string? namespaceStr, string? assemblyName, @@ -163,8 +164,8 @@ internal static Type CreateDerivedType(string name, ModuleBuilder moduleBuilder = GetModuleBuilder(assemblyName, moduleName); Type baseClass = baseType; - var interfaces = new List { typeof(IPythonDerivedType) }; - + var interfaces = new HashSet { typeof(IPythonDerivedType) }; + foreach(var t in interfaces2) interfaces.Add(t); // if the base type is an interface then use System.Object as the base class // and add the base type to the list of interfaces this new class will implement. if (baseType.IsInterface) @@ -215,7 +216,7 @@ internal static Type CreateDerivedType(string name, } // override any virtual methods not already overridden by the properties above - MethodInfo[] methods = baseType.GetMethods(); + var methods = baseType.GetMethods().Concat(interfaces.SelectMany(x => x.GetMethods())); var virtualMethods = new HashSet(); foreach (MethodInfo method in methods) { diff --git a/src/runtime/Types/MetaType.cs b/src/runtime/Types/MetaType.cs index db1a3ff2e..459796ebf 100644 --- a/src/runtime/Types/MetaType.cs +++ b/src/runtime/Types/MetaType.cs @@ -1,5 +1,8 @@ using System; +using System.Collections.Generic; using System.Diagnostics; +using System.Linq; +using System.Reflection; using System.Runtime.InteropServices; using System.Runtime.Serialization; @@ -84,36 +87,58 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, // That type must itself have a managed implementation. We check // that by making sure its metatype is the CLR metatype. - if (Runtime.PyTuple_Size(bases) != 1) + + List interfaces = new List(); + List baseType = new List(); + + var cnt = Runtime.PyTuple_GetSize(bases); + + for (uint i = 0; i < cnt; i++) + { + var base_type2 = Runtime.PyTuple_GetItem(bases, (int)i); + var cb2 = (ClassBase) GetManagedObject(base_type2); + if (cb2 != null) + { + if (cb2.type.Valid && cb2.type.Value.IsInterface) + interfaces.Add(cb2.type.Value); + else baseType.Add(cb2); + } + } + + if (baseType.Count == 0) + { + baseType.Add(new ClassBase(typeof(object))); + } + + + if (baseType.Count > 1) { return Exceptions.RaiseTypeError("cannot use multiple inheritance with managed classes"); } - BorrowedReference base_type = Runtime.PyTuple_GetItem(bases, 0); - BorrowedReference mt = Runtime.PyObject_TYPE(base_type); + /* + BorrowedReference mt = Runtime.PyObject_TYPE(baseType); if (!(mt == PyCLRMetaType || mt == Runtime.PyTypeType)) { return Exceptions.RaiseTypeError("invalid metatype"); - } + }*/ // Ensure that the reflected type is appropriate for subclassing, // disallowing subclassing of delegates, enums and array types. - if (GetManagedObject(base_type) is ClassBase cb) + var cb = baseType.First(); + try { - try + if (!cb.CanSubclass()) { - if (!cb.CanSubclass()) - { - return Exceptions.RaiseTypeError("delegates, enums and array types cannot be subclassed"); - } - } - catch (SerializationException) - { - return Exceptions.RaiseTypeError($"Underlying C# Base class {cb.type} has been deleted"); + return Exceptions.RaiseTypeError("delegates, enums and array types cannot be subclassed"); } } + catch (SerializationException) + { + return Exceptions.RaiseTypeError($"Underlying C# Base class {cb.type} has been deleted"); + } BorrowedReference slots = Runtime.PyDict_GetItem(dict, PyIdentifier.__slots__); if (slots != null) @@ -127,21 +152,25 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, // into python. if (null != dict) { + var btt = baseType.FirstOrDefault().type.ValueOrNull; + var ctor = btt?.GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .FirstOrDefault(x => x.GetParameters().Any() == false); using var clsDict = new PyDict(dict); if (clsDict.HasKey("__assembly__") || clsDict.HasKey("__namespace__") - || (cb.type.Valid && cb.type.Value != null && cb.type.Value.GetConstructor(Array.Empty()) != null)) + || (ctor != null)) { if (!clsDict.HasKey("__namespace__")) { clsDict["__namespace__"] = (clsDict["__module__"].ToString()).ToPython(); } - return TypeManager.CreateSubType(name, base_type, clsDict); + return TypeManager.CreateSubType(name, baseType, interfaces, clsDict); } } + var base_type = Runtime.PyTuple_GetItem(bases, 0); // otherwise just create a basic type without reflecting back into the managed side. IntPtr func = Util.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_new); NewReference type = NativeCall.Call_3(func, tp, args, kw); diff --git a/src/runtime/Types/ReflectedClrType.cs b/src/runtime/Types/ReflectedClrType.cs index b787939be..bbda4364f 100644 --- a/src/runtime/Types/ReflectedClrType.cs +++ b/src/runtime/Types/ReflectedClrType.cs @@ -68,7 +68,7 @@ internal void Restore(ClassBase cb) TypeManager.InitializeClass(this, cb, cb.type.Value); } - internal static NewReference CreateSubclass(ClassBase baseClass, + internal static NewReference CreateSubclass(ClassBase baseClass, IEnumerable interfaces, string name, string? assembly, string? ns, BorrowedReference dict) { @@ -76,6 +76,7 @@ internal static NewReference CreateSubclass(ClassBase baseClass, { Type subType = ClassDerivedObject.CreateDerivedType(name, baseClass.type.Value, + interfaces, dict, ns, assembly); diff --git a/src/testing/generictest.cs b/src/testing/generictest.cs index b333910c2..a78fd5104 100644 --- a/src/testing/generictest.cs +++ b/src/testing/generictest.cs @@ -95,6 +95,11 @@ public string Overloaded(int arg1, int arg2, string arg3) { return arg3; } + + public virtual Q VirtualOverloaded(Q arg) + { + return arg; + } } public class GenericStaticMethodTest @@ -118,10 +123,7 @@ public static Q Overloaded(Q arg) return arg; } - public static U Overloaded(Q arg1, U arg2) - { - return arg2; - } + public static string Overloaded(int arg1, int arg2, string arg3) { diff --git a/src/testing/subclasstest.cs b/src/testing/subclasstest.cs index ab0b73368..3ad34c6be 100644 --- a/src/testing/subclasstest.cs +++ b/src/testing/subclasstest.cs @@ -124,4 +124,25 @@ public static int test_event(IInterfaceTest x, int value) return et.value; } } + + public interface ISimpleInterface + { + bool Ok(); + } + + public class SimpleClass + { + + public static void TestObject(object obj) + { + if (obj is ISimpleInterface) + { + + } + else + { + throw new Exception(); + } + } + } } diff --git a/tests/test_subclass.py b/tests/test_subclass.py index 8b15d31ff..53e48f159 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -9,7 +9,7 @@ import System import pytest from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, - FunctionsTest, GenericVirtualMethodTest) + FunctionsTest, GenericVirtualMethodTest, ISimpleInterface, SimpleClass) from System.Collections.Generic import List @@ -272,6 +272,28 @@ class OverloadingSubclass2(OverloadingSubclass): obj = OverloadingSubclass() assert obj.VirtMethod[int](5) == 5 +def test_interface_and_class_impl(): + class OverloadingSubclass(GenericVirtualMethodTest): + __namespace__ = "test_virtual_generic_method_cls" + class OverloadingSubclass2(OverloadingSubclass): + pass + obj = OverloadingSubclass() + assert obj.VirtMethod[int](5) == 5 + +def test_interface_and_class_impl2(): + class DualSubClass(ISimpleInterface, SimpleClass): + def Ok(self): + return True + class DualSubClass2(ISimpleInterface): + def Ok(self): + return True + + obj = DualSubClass() + SimpleClass.TestObject(obj) + obj = DualSubClass2() + SimpleClass.TestObject(obj) + + def test_construction_from_clr(): import clr calls = [] From 6f0b35445467d65fe61c13d0c2afc3a3f22a5b46 Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Tue, 3 May 2022 22:46:03 +0200 Subject: [PATCH 05/17] Fixed issue with protected constructors and classes with no constructor --- src/runtime/Types/ClassDerived.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index a45beb78f..779bd0262 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -188,12 +188,17 @@ internal static Type CreateDerivedType(string name, FieldAttributes.Private); // override any constructors - ConstructorInfo[] constructors = baseClass.GetConstructors(); + ConstructorInfo[] constructors = baseClass.GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); foreach (ConstructorInfo ctor in constructors) { AddConstructor(ctor, baseType, typeBuilder); } + if (constructors.Length == 0) + { + AddConstructor(null, baseType, typeBuilder); + } + // Override any properties explicitly overridden in python var pyProperties = new HashSet(); if (py_dict != null && Runtime.PyDict_Check(py_dict)) @@ -303,7 +308,7 @@ internal static Type CreateDerivedType(string name, /// TypeBuilder for the new type the ctor is to be added to private static void AddConstructor(ConstructorInfo ctor, Type baseType, TypeBuilder typeBuilder) { - ParameterInfo[] parameters = ctor.GetParameters(); + ParameterInfo[] parameters = ctor?.GetParameters() ?? Array.Empty(); Type[] parameterTypes = (from param in parameters select param.ParameterType).ToArray(); // create a method for calling the original constructor @@ -322,14 +327,15 @@ private static void AddConstructor(ConstructorInfo ctor, Type baseType, TypeBuil { il.Emit(OpCodes.Ldarg, i + 1); } - il.Emit(OpCodes.Call, ctor); + if(ctor != null) + il.Emit(OpCodes.Call, ctor); il.Emit(OpCodes.Ret); // override the original method with a new one that dispatches to python ConstructorBuilder cb = typeBuilder.DefineConstructor(MethodAttributes.Public | MethodAttributes.ReuseSlot | MethodAttributes.HideBySig, - ctor.CallingConvention, + ctor?.CallingConvention ?? CallingConventions.Any, parameterTypes); il = cb.GetILGenerator(); il.DeclareLocal(typeof(object[])); From 3963621ba688bb6905df3ef392d6542c3865b156 Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Thu, 5 May 2022 19:26:30 +0200 Subject: [PATCH 06/17] Added supporr for class/property/method attributes. - Added shortened Attribute aliases which just generates tuples. - More general CLR base class support - creating a class instance from C# is now more similar to creating one from python. - Added attribute decorator to clr.py. - Added testing for the various possibilities --- src/python_tests_runner/PythonTestRunner.cs | 3 + src/runtime/Resources/clr.py | 67 +++++- src/runtime/Types/ClassDerived.cs | 231 +++++++++++++++++++- src/runtime/Types/ModuleObject.cs | 17 ++ src/testing/subclasstest.cs | 71 +++++- tests/test_subclass.py | 81 ++++++- 6 files changed, 454 insertions(+), 16 deletions(-) diff --git a/src/python_tests_runner/PythonTestRunner.cs b/src/python_tests_runner/PythonTestRunner.cs index 98b2fbe68..b4f030fd7 100644 --- a/src/python_tests_runner/PythonTestRunner.cs +++ b/src/python_tests_runner/PythonTestRunner.cs @@ -39,6 +39,9 @@ static IEnumerable PythonTestCases() yield return new[] { "test_delegate", "test_bool_delegate" }; yield return new[] { "test_subclass", "test_virtual_generic_method" }; yield return new[] { "test_subclass", "test_interface_and_class_impl2" }; + yield return new[] { "test_subclass", "test_class_with_attributes" }; + yield return new[] { "test_subclass", "test_class_with_advanced_attribute" }; + yield return new[] { "test_subclass", "test_more_subclasses" }; } /// diff --git a/src/runtime/Resources/clr.py b/src/runtime/Resources/clr.py index d4330a4d5..a5ff60370 100644 --- a/src/runtime/Resources/clr.py +++ b/src/runtime/Resources/clr.py @@ -21,16 +21,18 @@ def test(self): string z = x.test; // calls into python and returns "x" """ - def __init__(self, type_, fget=None, fset=None): + def __init__(self, type_, fget=None, fset=None, attributes = []): self.__name__ = getattr(fget, "__name__", None) self._clr_property_type_ = type_ self.fget = fget self.fset = fset - + self._clr_attributes_ = attributes def __call__(self, fget): - return self.__class__(self._clr_property_type_, + self.__class__(self._clr_property_type_, fget=fget, - fset=self.fset) + fset=self.fset, + attributes = self._clr_attributes_) + def setter(self, fset): self.fset = fset @@ -47,8 +49,32 @@ def __set__(self, instance, value): if not self.fset: raise AttributeError("%s is read-only" % self.__name__) return self.fset.__get__(instance, None)(value) + def add_attribute(self, attribute): + self._clr_attributes_.append(attribute) + return self +class property(object): + def __init__(self, type, default): + import weakref + self._clr_property_type_ = type + self.default = default + self.values = weakref.WeakKeyDictionary() + self._clr_attributes_ = [] + self.fget = 1 + self.fset = 1 + def __get__(self, instance, owner): + v = self.values.get(instance, self.default) + return v + def __set__(self, instance, value): + self.values[instance] = value + def add_attribute(self, attribute): + self._clr_attributes_.append(attribute) + return self + def __call__(self, type, default): + self2 = self.__class__(self._clr_property_type_, type, default) + self2._clr_attributes_ = self._clr_attributes_ + return self2 class clrmethod(object): """ Method decorator for exposing python methods to .NET. @@ -67,18 +93,47 @@ def test(self, x): int z = x.test("hello"); // calls into python and returns len("hello") """ - def __init__(self, return_type, arg_types, clrname=None, func=None): + def __init__(self, return_type = None, arg_types = [], clrname=None, func=None, **kwargs): + if return_type == None: + import System + return_type = System.Void self.__name__ = getattr(func, "__name__", None) self._clr_return_type_ = return_type self._clr_arg_types_ = arg_types self._clr_method_name_ = clrname or self.__name__ self.__func = func + if 'attributes' in kwargs: + self._clr_attributes_ = kwargs["attributes"] + else: + self._clr_attributes_ = [] def __call__(self, func): - return self.__class__(self._clr_return_type_, + self2 = self.__class__(self._clr_return_type_, self._clr_arg_types_, clrname=self._clr_method_name_, func=func) + self2._clr_attributes_ = self._clr_attributes_ + return self2 def __get__(self, instance, owner): return self.__func.__get__(instance, owner) + + def clr_attribute(self, attribute): + self._clr_attributes_.append(attribute) + return self + +class attribute(object): + + def __init__(self, attr, *args, **kwargs): + self.attr = attr + import Python.Runtime + #todo: ensure that attributes only are pushed when @ is used. + #import inspect + #Python.Runtime.PythonDerivedType.Test(inspect.stack()[1].code_context) + + Python.Runtime.PythonDerivedType.PushAttribute(attr) + def __call__(self, x): + import Python.Runtime + if Python.Runtime.PythonDerivedType.AssocAttribute(self.attr, x): + pass + return x diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 779bd0262..6404b7460 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -137,6 +137,112 @@ internal static NewReference ToPython(IPythonDerivedType obj) return result; } + static CustomAttributeBuilder AddAttribute(PyObject attr) + { + // this is a bit complicated because we want to support both unnamed and named arguments + // in C# there is a discrepancy between property setters and named argument wrt attributes + // in python there is no difference. But luckily there is rarely a conflict, so we can treat + // named arguments and named properties or fields the same way. + var tp = new PyTuple(attr); + Type attribute = (Type) tp[0].AsManagedObject(typeof(object)); + + var args = (PyObject[]) tp[1].AsManagedObject(typeof(PyObject[])); + var dict = new PyDict(tp[2]); + var dict2 = new Dictionary(); + foreach (var key in dict.Keys()) + { + var k = key.As(); + dict2[k] = dict[key]; + } + + // todo support kwargs and tupleargs + var allconstructors = attribute.GetConstructors(); + foreach (var constructor in allconstructors) + { + var parameters = constructor.GetParameters(); + List paramValues = new List(); + HashSet accountedFor = new HashSet(); + for (int i = 0; i < parameters.Length; i++) + { + var parameter = parameters[i]; + if (parameter.ParameterType.IsArray && null != parameter.GetCustomAttribute(typeof(ParamArrayAttribute))) + { + int cnt = args.Length - i; + var values = Array.CreateInstance(parameter.ParameterType.GetElementType(), cnt); + for(int j = 0; j < cnt; j++) + values.SetValue(args[i + j], j); + paramValues.Add(values); + break; + } + + PyObject argvalue = null; + if (args.Length <= i && dict2.TryGetValue(parameter.Name, out argvalue)) + { + accountedFor.Add(parameter.Name); + }else if (args.Length <= i && parameter.IsOptional) + { + paramValues.Add(parameter.DefaultValue); + continue; + }else if (args.Length <= i) + { + goto next; + } + else + { + argvalue = args[i]; + } + + var argval = argvalue.AsManagedObject(parameter.ParameterType); + if (parameter.ParameterType.IsAssignableFrom(argval?.GetType() ?? typeof(object))) + paramValues.Add(argval); + else if (parameter.IsOptional) + { + paramValues.Add(parameter.DefaultValue); + } + } + + List namedProperties = new List(); + List namedPropertyValues = new List(); + List namedFields = new List(); + List namedFieldValues = new List(); + + foreach (var key in dict2.Keys.Where(x => accountedFor.Contains(x) == false)) + { + var member = attribute.GetMember(key).FirstOrDefault(); + if (member == null) + goto next; + if (member is PropertyInfo prop) + { + namedProperties.Add(prop); + var argval = dict2[key].AsManagedObject(prop.PropertyType); + if (prop.PropertyType.IsAssignableFrom(argval?.GetType() ?? typeof(object))) + namedPropertyValues.Add(argval); + else + goto next; + } + if (member is FieldInfo field) + { + namedFields.Add(field); + var argval = dict2[key].AsManagedObject(field.FieldType); + if (field.FieldType.IsAssignableFrom(argval?.GetType() ?? typeof(object))) + namedFieldValues.Add(argval); + else + goto next; + } + } + + var cb = new CustomAttributeBuilder(constructor, paramValues.ToArray(), + namedProperties.ToArray(), namedPropertyValues.ToArray(), + namedFields.ToArray(), namedFieldValues.ToArray()); + return cb; + next: ; + } + + return null; + } + + + /// /// Creates a new managed type derived from a base type with any virtual /// methods overridden to call out to python if the associated python @@ -181,11 +287,14 @@ internal static Type CreateDerivedType(string name, // add a field for storing the python object pointer // FIXME: fb not used - FieldBuilder fb = typeBuilder.DefineField(PyObjName, + if (baseClass.GetField(PyObjName, PyObjFlags) == null) + { + FieldBuilder fb = typeBuilder.DefineField(PyObjName, #pragma warning disable CS0618 // Type or member is obsolete. OK for internal use. - typeof(UnsafeReferenceWithRun), + typeof(UnsafeReferenceWithRun), #pragma warning restore CS0618 // Type or member is obsolete - FieldAttributes.Private); + FieldAttributes.Public); + } // override any constructors ConstructorInfo[] constructors = baseClass.GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); @@ -250,6 +359,24 @@ internal static Type CreateDerivedType(string name, if (py_dict != null && Runtime.PyDict_Check(py_dict)) { using var dict = new PyDict(py_dict); + if (dict.HasKey("__clr_attributes__")) + { + var attributes = new PyList(dict["__clr_attributes__"]); + foreach (var attr in attributes) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + typeBuilder.SetCustomAttribute(builder); + } + } + + foreach (var attr in PopAttributes()) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + typeBuilder.SetCustomAttribute(builder); + } + using var keys = dict.Keys(); foreach (PyObject pyKey in keys) { @@ -290,6 +417,7 @@ internal static Type CreateDerivedType(string name, Type type = typeBuilder.CreateType(); + // scan the assembly so the newly added class can be imported Assembly assembly = Assembly.GetAssembly(type); AssemblyManager.ScanAssembly(assembly); @@ -300,6 +428,8 @@ internal static Type CreateDerivedType(string name, return type; } + private static Dictionary pyTypeLookup = new Dictionary(); + /// /// Add a constructor override that calls the python ctor after calling the base type constructor. /// @@ -478,7 +608,8 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde } using var pyReturnType = func.GetAttr("_clr_return_type_"); - using var pyArgTypes = func.GetAttr("_clr_arg_types_"); + usingvar attributes = new PyList(func.GetAttr("_clr_attributes_"))) + using (var pyArgTypes = func.GetAttr("_clr_arg_types_"); using var pyArgTypesIter = PyIter.GetIter(pyArgTypes); var returnType = pyReturnType.AsManagedObject(typeof(Type)) as Type; if (returnType == null) @@ -509,7 +640,27 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde returnType, argTypes.ToArray()); - ILGenerator il = methodBuilder.GetILGenerator(); + foreach (var attr in attributes) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + methodBuilder.SetCustomAttribute(builder); + } + + if (methodAssoc.TryGetValue(func, out var lst)) + { + foreach(var attr in lst) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + methodBuilder.SetCustomAttribute(builder); + } + + methodAssoc.Remove(func); + + } + + ILGenerator il = methodBuilder.GetILGenerator(); il.DeclareLocal(typeof(object[])); il.DeclareLocal(typeof(RuntimeMethodHandle)); @@ -600,6 +751,18 @@ private static void AddPythonProperty(string propertyName, PyObject func, TypeBu PropertyAttributes.None, propertyType, null); + if (func.HasAttr("_clr_attributes_")) + { + using (var attributes = new PyList(func.GetAttr("_clr_attributes_"))) + { + foreach (var attr in attributes) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + propertyBuilder.SetCustomAttribute(builder); + } + } + } if (func.HasAttr("fget")) { @@ -695,8 +858,45 @@ private static ModuleBuilder GetModuleBuilder(string assemblyName, string module [Obsolete(Util.InternalUseOnly)] public class PythonDerivedType { + private static List attributesStack = new List(); + internal static Dictionary> methodAssoc = new Dictionary>(); + public static void PushAttribute(PyObject obj) + { + var tp = new PyTuple(obj); + attributesStack.Add(tp); + } + + public static bool AssocAttribute(PyObject obj, PyObject func) + { + using var _ = Py.GIL(); + var tp = new PyTuple(obj); + for (int i = 0; i < attributesStack.Count; i++) + { + if (Equals(tp, attributesStack[i])) + { + attributesStack.RemoveAt(i); + if (!methodAssoc.TryGetValue(func, out var lst)) + { + lst = methodAssoc[func] = new List(); + } + lst.Add(tp); + return true; + } + } + return false; + + } + + + public static IEnumerable PopAttributes() + { + if (attributesStack.Count == 0) return Array.Empty(); + var attrs = attributesStack; + attributesStack = new List(); + return attrs; + } internal const string PyObjName = "__pyobj__"; - internal const BindingFlags PyObjFlags = BindingFlags.Instance | BindingFlags.NonPublic; + internal const BindingFlags PyObjFlags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.FlattenHierarchy; /// /// This is the implementation of the overridden methods in the derived @@ -896,6 +1096,7 @@ public static void InvokeSetProperty(IPythonDerivedType obj, string propertyN } } + static private PyObject pin; public static void InvokeCtor(IPythonDerivedType obj, string origCtorName, object[] args) { var selfRef = GetPyObj(obj); @@ -906,8 +1107,22 @@ public static void InvokeCtor(IPythonDerivedType obj, string origCtorName, objec // In the end we decrement the python object's reference count. // This doesn't actually destroy the object, it just sets the reference to this object // to be a weak reference and it will be destroyed when the C# object is destroyed. - using var self = CLRObject.GetReference(obj, obj.GetType()); - SetPyObj(obj, self.Borrow()); + + PyType cc = ClassManager.GetClass(obj.GetType()); + var args2 = new PyObject[args.Length]; + for (int i = 0; i < args.Length; i++) + args2[i] = args[i].ToPython(); + // create an instance of the class and steal the reference. + using (var obj2 = cc.Invoke(args2, null)) + { + // somehow if this is not done this object never gets a reference count + // and things breaks later. + // I am not sure what it does though. + GCHandle gc = GCHandle.Alloc(obj); + // hand over the reference. + var py = obj2.NewReferenceOrNull(); + SetPyObj(obj, py.Borrow()); + } } // call the base constructor diff --git a/src/runtime/Types/ModuleObject.cs b/src/runtime/Types/ModuleObject.cs index a9e8c9937..545e5851d 100644 --- a/src/runtime/Types/ModuleObject.cs +++ b/src/runtime/Types/ModuleObject.cs @@ -133,6 +133,23 @@ public NewReference GetAttribute(string name, bool guess) return new NewReference(c); } + // attribute names + // for attributes without the Attribute suffix, create an attribute builder. + var qname2 = qname + "Attribute"; + var type2 = AssemblyManager.LookupTypes(qname2).FirstOrDefault(t => t.IsPublic); + if (type2 != null) + { + var str = "def {2}attrbuilder(*args, **kwargs):\n" + + " import {1}\n" + + " return ({0}, args, kwargs)\n"; + str = string.Format(str, qname2, _namespace, name); + PythonEngine.Exec(str); + var obj = PythonEngine.Eval(name + "attrbuilder"); + var o = obj.NewReferenceOrNull(); + this.StoreAttribute(name, o.Borrow()); + return new NewReference(o); + } + // We didn't find the name, so we may need to see if there is a // generic type with this base name. If so, we'll go ahead and // return it. Note that we store the mapping of the unmangled diff --git a/src/testing/subclasstest.cs b/src/testing/subclasstest.cs index 3ad34c6be..989929ea3 100644 --- a/src/testing/subclasstest.cs +++ b/src/testing/subclasstest.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Threading; namespace Python.Test { @@ -130,13 +131,42 @@ public interface ISimpleInterface bool Ok(); } + public class TestAttributeAttribute: Attribute + { + public int X { get; set; } + public int Y { get; set; } + public string Z { get; set; } + public string W { get; set; } + public TestAttributeAttribute(int x, int y, string z = "x") + { + X = x; + Y = y; + Z = z; + + } + } + + public class SimpleClass { + public bool Initialized; + + public SimpleClass() + { + Initialized = true; + } + private int counter = 0; + public virtual int IncrementThing() + { + return ++counter; + } public static void TestObject(object obj) { - if (obj is ISimpleInterface) + if (obj is ISimpleInterface si) { + if (!si.Ok()) + throw new Exception(); } else @@ -144,5 +174,44 @@ public static void TestObject(object obj) throw new Exception(); } } + public static void TestObjectProperty(object obj, string prop, double newval) + { + obj.GetType().GetProperty(prop).SetValue(obj, newval); + var val = obj.GetType().GetProperty(prop).GetValue(obj); + if (!Equals(newval, val)) + throw new Exception(); + } + + private static SimpleClass objStore; + public static void Test1(SimpleClass obj) + { + objStore = obj; + int x = obj.IncrementThing(); + } + + public static void Test2() + { + GC.Collect(); + + var threads = new Thread[20]; + for(int i = 0; i < threads.Length; i++) + threads[i] = new Thread(() => TestObjectProperty(objStore, "X", 10.0)); + for (int i = 0; i < threads.Length; i++) + threads[i].Start(); + for (int i = 0; i < threads.Length; i++) + threads[i].Join(); + } + + public static object InvokeCtor(Type t) + { + var obj = Activator.CreateInstance(t); + return obj; + } + + public static void Pause() + { + + } + } } diff --git a/tests/test_subclass.py b/tests/test_subclass.py index 53e48f159..828435183 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -7,9 +7,12 @@ """Test sub-classing managed types""" import System +from System import (Console, Attribute, Double) +from System.Diagnostics import (DebuggerDisplay, DebuggerDisplayAttribute, Debug) +from System.ComponentModel import (Browsable, BrowsableAttribute) import pytest from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, - FunctionsTest, GenericVirtualMethodTest, ISimpleInterface, SimpleClass) + FunctionsTest, GenericVirtualMethodTest, ISimpleInterface, SimpleClass, TestAttribute) from System.Collections.Generic import List @@ -293,6 +296,82 @@ def Ok(self): obj = DualSubClass2() SimpleClass.TestObject(obj) +def test_class_with_attributes(): + import clr + @clr.attribute(Browsable(False)) + class ClassWithAttributes(ISimpleInterface, SimpleClass): + __clr_attributes__ = [DebuggerDisplay("X: {X}")] + @clr.attribute(Browsable(True)) + def Ok(self): + return True + @clr.attribute(Browsable(True)) + @clr.clrmethod(int, [int]) + def Method1(x): + return x + + X = clr.property(Double, 1.0).add_attribute(DebuggerDisplay("Asd")) + obj = ClassWithAttributes() + tp = obj.GetType() + founddisplay = 0 + foundbrowsable = 0 + for attr in Attribute.GetCustomAttributes(tp): + if isinstance(attr, DebuggerDisplayAttribute): + founddisplay = founddisplay + 1 + if isinstance(attr, BrowsableAttribute): + foundbrowsable = foundbrowsable + 1 + SimpleClass.TestObject(obj) + found_display_on_property = 0 + for attr in Attribute.GetCustomAttributes(tp.GetProperty("X")): + if isinstance(attr, DebuggerDisplayAttribute): + found_display_on_property = found_display_on_property + 1 + found_display_on_method = 0 + for attr in Attribute.GetCustomAttributes(tp.GetMethod("Method1")): + if isinstance(attr, BrowsableAttribute): + found_display_on_method = found_display_on_method + 1 + assert founddisplay == 1 + assert found_display_on_property == 1 + assert found_display_on_method == 1 + assert foundbrowsable == 1 + assert obj.X == 1.0 + SimpleClass.TestObjectProperty(obj, "X", 10.0) +def test_class_with_advanced_attribute(): + import clr + @clr.attribute(TestAttribute(1, 2, z = "A", W = "B")) + class ClassWithAttributes2(ISimpleInterface, SimpleClass): + pass + c = ClassWithAttributes2() +def test_more_subclasses(): + import clr + class SubClass1(SimpleClass): + X = clr.property(Double, 1.0) + def __init__(self): + super().__init__() + self.Y = 10.0 + SimpleClass.Pause(); + + @clr.attribute(DebuggerDisplay("X")) + + class SubClass2(SubClass1): + __namespace__ = "TestModule" + def __init__(self): + SimpleClass.Pause(); + super().__init__() + def IncrementThing(self): + return 6; + obj = SimpleClass.InvokeCtor(SubClass2) + + obj2 = SubClass2() + tp = obj.GetType() + obj.X = 5.0 + assert obj.Y == 10.0 + assert obj2.Y == 10.0 + assert obj.Initialized == True + assert obj2.Initialized == True + SimpleClass.Test1(obj) + obj = None + SimpleClass.Test2() + + def test_construction_from_clr(): import clr From 4e0d00386092b4cce3f73eb0d80b8274413dc960 Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Thu, 5 May 2022 22:14:39 +0200 Subject: [PATCH 07/17] - Expanded the way attributes amy be added - Fixed search bug in AssocAttribute - Added testing --- src/runtime/Resources/clr.py | 51 +++++++++++++++++++++++-------- src/runtime/Types/ClassDerived.cs | 3 +- tests/test_subclass.py | 7 ++++- 3 files changed, 46 insertions(+), 15 deletions(-) diff --git a/src/runtime/Resources/clr.py b/src/runtime/Resources/clr.py index a5ff60370..0f412452d 100644 --- a/src/runtime/Resources/clr.py +++ b/src/runtime/Resources/clr.py @@ -49,8 +49,14 @@ def __set__(self, instance, value): if not self.fset: raise AttributeError("%s is read-only" % self.__name__) return self.fset.__get__(instance, None)(value) - def add_attribute(self, attribute): - self._clr_attributes_.append(attribute) + def add_attribute(self, *args, **kwargs): + lst = [] + if len(args) > 0: + if isinstance(args[0], tuple): + lst = args + else: + lst = [(args[0], args[1:], kwargs)] + self._clr_attributes_.extend(lst) return self class property(object): @@ -68,9 +74,16 @@ def __get__(self, instance, owner): return v def __set__(self, instance, value): self.values[instance] = value - def add_attribute(self, attribute): - self._clr_attributes_.append(attribute) + def add_attribute(self, *args, **kwargs): + lst = [] + if len(args) > 0: + if isinstance(args[0], tuple): + lst = args + else: + lst = [(args[0], args[1:], kwargs)] + self._clr_attributes_.extend(lst) return self + def __call__(self, type, default): self2 = self.__class__(self._clr_property_type_, type, default) self2._clr_attributes_ = self._clr_attributes_ @@ -118,22 +131,34 @@ def __call__(self, func): def __get__(self, instance, owner): return self.__func.__get__(instance, owner) - def clr_attribute(self, attribute): - self._clr_attributes_.append(attribute) + def add_attribute(self, *args, **kwargs): + lst = [] + if len(args) > 0: + if isinstance(args[0], tuple): + lst = args + else: + lst = [(args[0], args[1:], kwargs)] + self._clr_attributes_.extend(lst) return self class attribute(object): - def __init__(self, attr, *args, **kwargs): - self.attr = attr + def __init__(self, *args, **kwargs): + lst = [] + if len(args) > 0: + if isinstance(args[0], tuple): + lst = args + else: + lst = [(args[0], args[1:], kwargs)] import Python.Runtime #todo: ensure that attributes only are pushed when @ is used. - #import inspect - #Python.Runtime.PythonDerivedType.Test(inspect.stack()[1].code_context) + self.attr = lst + for item in lst: + Python.Runtime.PythonDerivedType.PushAttribute(item) - Python.Runtime.PythonDerivedType.PushAttribute(attr) def __call__(self, x): import Python.Runtime - if Python.Runtime.PythonDerivedType.AssocAttribute(self.attr, x): - pass + for item in self.attr: + if Python.Runtime.PythonDerivedType.AssocAttribute(item, x): + pass return x diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 6404b7460..0eaf70524 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -872,7 +872,8 @@ public static bool AssocAttribute(PyObject obj, PyObject func) var tp = new PyTuple(obj); for (int i = 0; i < attributesStack.Count; i++) { - if (Equals(tp, attributesStack[i])) + + if (tp.BorrowNullable()== attributesStack[i].BorrowNullable()) { attributesStack.RemoveAt(i); if (!methodAssoc.TryGetValue(func, out var lst)) diff --git a/tests/test_subclass.py b/tests/test_subclass.py index 828435183..e0fe5f647 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -12,7 +12,7 @@ from System.ComponentModel import (Browsable, BrowsableAttribute) import pytest from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, - FunctionsTest, GenericVirtualMethodTest, ISimpleInterface, SimpleClass, TestAttribute) + FunctionsTest, GenericVirtualMethodTest, ISimpleInterface, SimpleClass, TestAttribute, TestAttributeAttribute) from System.Collections.Generic import List @@ -339,7 +339,12 @@ def test_class_with_advanced_attribute(): @clr.attribute(TestAttribute(1, 2, z = "A", W = "B")) class ClassWithAttributes2(ISimpleInterface, SimpleClass): pass + @clr.attribute(TestAttributeAttribute, 1, 2, z = "A", W = "B") + class ClassWithAttributes3(ISimpleInterface, SimpleClass): + X = clr.property(Double, 1.0).add_attribute(TestAttributeAttribute, 1, 2) + c = ClassWithAttributes2() + c2 = ClassWithAttributes3() def test_more_subclasses(): import clr class SubClass1(SimpleClass): From ea598a21aba69cbfbdf863b3899da814b8269a3e Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Fri, 6 May 2022 08:30:13 +0200 Subject: [PATCH 08/17] added support for creating abstract classes using a __clr_abstract__ attribute. --- src/python_tests_runner/PythonTestRunner.cs | 1 + src/runtime/Types/ClassDerived.cs | 11 ++++++++++- tests/test_subclass.py | 10 +++++++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/python_tests_runner/PythonTestRunner.cs b/src/python_tests_runner/PythonTestRunner.cs index b4f030fd7..58b113e33 100644 --- a/src/python_tests_runner/PythonTestRunner.cs +++ b/src/python_tests_runner/PythonTestRunner.cs @@ -42,6 +42,7 @@ static IEnumerable PythonTestCases() yield return new[] { "test_subclass", "test_class_with_attributes" }; yield return new[] { "test_subclass", "test_class_with_advanced_attribute" }; yield return new[] { "test_subclass", "test_more_subclasses" }; + yield return new[] { "test_subclass", "abstract_test" }; } /// diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 0eaf70524..a819c0e7d 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -280,8 +280,16 @@ internal static Type CreateDerivedType(string name, baseClass = typeof(object); } + bool isAbstract = false; + if (py_dict != null && Runtime.PyDict_Check(py_dict)) + { + using var dict = new PyDict(py_dict); + if (dict.HasKey("__clr_abstract__")) + isAbstract = true; + } + TypeBuilder typeBuilder = moduleBuilder.DefineType(name, - TypeAttributes.Public | TypeAttributes.Class, + TypeAttributes.Public | TypeAttributes.Class | (isAbstract ? TypeAttributes.Abstract : 0), baseClass, interfaces.ToArray()); @@ -359,6 +367,7 @@ internal static Type CreateDerivedType(string name, if (py_dict != null && Runtime.PyDict_Check(py_dict)) { using var dict = new PyDict(py_dict); + if (dict.HasKey("__clr_attributes__")) { var attributes = new PyList(dict["__clr_attributes__"]); diff --git a/tests/test_subclass.py b/tests/test_subclass.py index e0fe5f647..0aac33481 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -376,7 +376,15 @@ def IncrementThing(self): obj = None SimpleClass.Test2() - +def abstract_test(): + class abstractClass(SimpleClass): + __clr_abstract__ = True + failed = False + try: + abstractClass() + except: + failed = True + assert failed def test_construction_from_clr(): import clr From d6abbb2a312e7810698a2d0bc23644bd6578b621 Mon Sep 17 00:00:00 2001 From: Rolf Madsen Date: Tue, 24 May 2022 10:45:43 +0200 Subject: [PATCH 09/17] Improved AttributeError when looking for a symbol that does not exist inside a .NET module. --- src/python_tests_runner/PythonTestRunner.cs | 2 +- src/runtime/Types/ClassDerived.cs | 1 + src/runtime/Types/ModuleObject.cs | 2 +- src/testing/subclasstest.cs | 9 ++++++++- tests/test_subclass.py | 18 +++++++++++++++++- 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/python_tests_runner/PythonTestRunner.cs b/src/python_tests_runner/PythonTestRunner.cs index 58b113e33..9562c6a8d 100644 --- a/src/python_tests_runner/PythonTestRunner.cs +++ b/src/python_tests_runner/PythonTestRunner.cs @@ -18,7 +18,7 @@ public class PythonTestRunner public void SetUp() { Python.Runtime.Runtime.PythonDLL = - "/Library/Frameworks/Python.framework/Versions/3.10/lib/libpython3.10.dylib"; + "C:\\Python37.2\\python37.dll"; PythonEngine.Initialize(); } diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index a819c0e7d..726e4e41f 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -871,6 +871,7 @@ public class PythonDerivedType internal static Dictionary> methodAssoc = new Dictionary>(); public static void PushAttribute(PyObject obj) { + using var _ = Py.GIL(); var tp = new PyTuple(obj); attributesStack.Add(tp); } diff --git a/src/runtime/Types/ModuleObject.cs b/src/runtime/Types/ModuleObject.cs index 545e5851d..db9532dcd 100644 --- a/src/runtime/Types/ModuleObject.cs +++ b/src/runtime/Types/ModuleObject.cs @@ -340,7 +340,7 @@ public static NewReference tp_getattro(BorrowedReference ob, BorrowedReference k if (attr.IsNull()) { - Exceptions.SetError(Exceptions.AttributeError, name); + Exceptions.SetError(Exceptions.AttributeError, $"name '{name}' is not defined in module '{self.moduleName}'."); return default; } diff --git a/src/testing/subclasstest.cs b/src/testing/subclasstest.cs index 989929ea3..cdfee3aed 100644 --- a/src/testing/subclasstest.cs +++ b/src/testing/subclasstest.cs @@ -130,7 +130,10 @@ public interface ISimpleInterface { bool Ok(); } - + public interface ISimpleInterface2 + { + int Execute(CancellationToken token); + } public class TestAttributeAttribute: Attribute { public int X { get; set; } @@ -168,6 +171,10 @@ public static void TestObject(object obj) if (!si.Ok()) throw new Exception(); + }else if (obj is ISimpleInterface2 si2) + { + si2.Execute(CancellationToken.None); + } else { diff --git a/tests/test_subclass.py b/tests/test_subclass.py index 0aac33481..e941d0691 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -10,9 +10,11 @@ from System import (Console, Attribute, Double) from System.Diagnostics import (DebuggerDisplay, DebuggerDisplayAttribute, Debug) from System.ComponentModel import (Browsable, BrowsableAttribute) +from System.Threading import (CancellationToken) import pytest from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, - FunctionsTest, GenericVirtualMethodTest, ISimpleInterface, SimpleClass, TestAttribute, TestAttributeAttribute) + FunctionsTest, GenericVirtualMethodTest, ISimpleInterface, SimpleClass, TestAttribute, TestAttributeAttribute, ISimpleInterface2) +import Python.Test from System.Collections.Generic import List @@ -290,12 +292,26 @@ def Ok(self): class DualSubClass2(ISimpleInterface): def Ok(self): return True + class DualSubClass3(ISimpleInterface2): + def Execute(self, cancellationToken): + return 0 + try: + class DualSubClass4(Python.Test.ISimpleInterface3): + def Execute(self, cancellationToken): + return 0 + assert False # An exception should be thrown. + except AttributeError as ae: + assert ("not defined" in str(ae)) obj = DualSubClass() SimpleClass.TestObject(obj) obj = DualSubClass2() SimpleClass.TestObject(obj) + obj2 = DualSubClass3(); + SimpleClass.TestObject(obj2) + #obj2.Execute(CancellationToken.None) + def test_class_with_attributes(): import clr @clr.attribute(Browsable(False)) From 2a84675fbdebfcda47e4d05d09af328c6b44967d Mon Sep 17 00:00:00 2001 From: romadsen-ks Date: Tue, 28 Jun 2022 11:55:28 +0200 Subject: [PATCH 10/17] Added test to detect an issue with object construction. --- src/testing/subclasstest.cs | 26 ++++++++++++++++++++++++++ tests/test_subclass.py | 23 ++++++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/testing/subclasstest.cs b/src/testing/subclasstest.cs index cdfee3aed..943366499 100644 --- a/src/testing/subclasstest.cs +++ b/src/testing/subclasstest.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.Threading; +using Python.Runtime; + namespace Python.Test { public interface IInterfaceTest @@ -215,6 +217,30 @@ public static object InvokeCtor(Type t) return obj; } + public object TestObj { get; set; } + + public static object TestOnType(Type t) + { + using (Py.GIL()) + { + var obj = (SimpleClass) Activator.CreateInstance(t); + //obj = obj.ToPython().As(); + obj.TestObj = new object(); + var py = obj.ToPython(); + var man = py.As(); + if (!ReferenceEquals(man, obj)) + throw new Exception("Same object expected"); + var setObj = py.GetAttr("TestObj").As(); + if (setObj == null) + throw new NullReferenceException(); + if (ReferenceEquals(setObj, obj.TestObj) == false) + throw new Exception("!!"); + + + return obj; + } + } + public static void Pause() { diff --git a/tests/test_subclass.py b/tests/test_subclass.py index e941d0691..585947d56 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -361,9 +361,27 @@ class ClassWithAttributes3(ISimpleInterface, SimpleClass): c = ClassWithAttributes2() c2 = ClassWithAttributes3() + +def test_subclass_ctor(): + import clr + class SubClass0(SimpleClass): + pass + class SubClass1(SubClass0): + def __init__(self): + super().__init__() + class SubClass2(SubClass1): + __namespace__ = "TestModule" + def __init__(self): + super().__init__() + SimpleClass.TestOnType(SubClass0) + SimpleClass.TestOnType(SubClass1) + SimpleClass.TestOnType(SubClass2) + def test_more_subclasses(): import clr - class SubClass1(SimpleClass): + class SubClass0(SimpleClass): + pass + class SubClass1(SubClass0): X = clr.property(Double, 1.0) def __init__(self): super().__init__() @@ -379,6 +397,9 @@ def __init__(self): super().__init__() def IncrementThing(self): return 6; + SimpleClass.TestOnType(SubClass0) + SimpleClass.TestOnType(SubClass1) + SimpleClass.TestOnType(SubClass2) obj = SimpleClass.InvokeCtor(SubClass2) obj2 = SubClass2() From 172c64249d37c12ef96f538900d4658e7dd0be0a Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Thu, 30 Jun 2022 16:25:15 -0700 Subject: [PATCH 11/17] got rid of a few deprecation warnings that pollute GitHub code review --- src/embed_tests/TestConverter.cs | 9 ++++++--- src/embed_tests/TestDomainReload.cs | 3 +-- src/embed_tests/TestFinalizer.cs | 7 +++++-- src/embed_tests/TestNativeTypeOffset.cs | 3 ++- src/embed_tests/TestPythonException.cs | 2 +- src/runtime/InternString.cs | 4 ++-- src/runtime/PythonTypes/PyObject.cs | 4 +++- src/runtime/Types/ReflectedClrType.cs | 2 +- src/runtime/Util/PythonReferenceComparer.cs | 4 ++-- 9 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/embed_tests/TestConverter.cs b/src/embed_tests/TestConverter.cs index e586eda1b..0686d528b 100644 --- a/src/embed_tests/TestConverter.cs +++ b/src/embed_tests/TestConverter.cs @@ -148,7 +148,7 @@ public void PyIntImplicit() { var i = new PyInt(1); var ni = (PyObject)i.As(); - Assert.AreEqual(i.rawPtr, ni.rawPtr); + Assert.IsTrue(PythonReferenceComparer.Instance.Equals(i, ni)); } [Test] @@ -178,8 +178,11 @@ public void RawPyObjectProxy() var clrObject = (CLRObject)ManagedType.GetManagedObject(pyObjectProxy); Assert.AreSame(pyObject, clrObject.inst); - var proxiedHandle = pyObjectProxy.GetAttr("Handle").As(); - Assert.AreEqual(pyObject.Handle, proxiedHandle); +#pragma warning disable CS0612 // Type or member is obsolete + const string handlePropertyName = nameof(PyObject.Handle); +#pragma warning restore CS0612 // Type or member is obsolete + var proxiedHandle = pyObjectProxy.GetAttr(handlePropertyName).As(); + Assert.AreEqual(pyObject.DangerousGetAddressOrNull(), proxiedHandle); } // regression for https://github.com/pythonnet/pythonnet/issues/451 diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 498119d1e..a0f9b63eb 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -99,8 +99,7 @@ from Python.EmbeddingTest.Domain import MyClass { Debug.Assert(obj.AsManagedObject(type).GetType() == type); // We only needs its Python handle - PyRuntime.XIncref(obj); - return obj.Handle; + return new NewReference(obj).DangerousMoveToPointer(); } } } diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 40ab03395..b748a2244 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -212,7 +212,9 @@ public void ValidateRefCount() Assert.AreEqual(ptr, e.Handle); Assert.AreEqual(2, e.ImpactedObjects.Count); // Fix for this test, don't do this on general environment +#pragma warning disable CS0618 // Type or member is obsolete Runtime.Runtime.XIncref(e.Reference); +#pragma warning restore CS0618 // Type or member is obsolete return false; }; Finalizer.Instance.IncorrectRefCntResolver += handler; @@ -234,8 +236,9 @@ private static IntPtr CreateStringGarbage() { PyString s1 = new PyString("test_string"); // s2 steal a reference from s1 - PyString s2 = new PyString(StolenReference.DangerousFromPointer(s1.Handle)); - return s1.Handle; + IntPtr address = s1.Reference.DangerousGetAddress(); + PyString s2 = new (StolenReference.DangerousFromPointer(address)); + return address; } } } diff --git a/src/embed_tests/TestNativeTypeOffset.cs b/src/embed_tests/TestNativeTypeOffset.cs index 2d31fe506..d692c24e6 100644 --- a/src/embed_tests/TestNativeTypeOffset.cs +++ b/src/embed_tests/TestNativeTypeOffset.cs @@ -33,7 +33,8 @@ public void LoadNativeTypeOffsetClass() { PyObject sys = Py.Import("sys"); // We can safely ignore the "m" abi flag - var abiflags = sys.GetAttr("abiflags", "".ToPython()).ToString().Replace("m", ""); + var abiflags = sys.HasAttr("abiflags") ? sys.GetAttr("abiflags").ToString() : ""; + abiflags = abiflags.Replace("m", ""); if (!string.IsNullOrEmpty(abiflags)) { string typeName = "Python.Runtime.NativeTypeOffset, Python.Runtime"; diff --git a/src/embed_tests/TestPythonException.cs b/src/embed_tests/TestPythonException.cs index a7cf05c83..a248b6a1f 100644 --- a/src/embed_tests/TestPythonException.cs +++ b/src/embed_tests/TestPythonException.cs @@ -161,7 +161,7 @@ def __init__(self, val): using var tbObj = tbPtr.MoveToPyObject(); // the type returned from PyErr_NormalizeException should not be the same type since a new // exception was raised by initializing the exception - Assert.AreNotEqual(type.Handle, typeObj.Handle); + Assert.IsFalse(PythonReferenceComparer.Instance.Equals(type, typeObj)); // the message should now be the string from the throw exception during normalization Assert.AreEqual("invalid literal for int() with base 10: 'dummy string'", strObj.ToString()); } diff --git a/src/runtime/InternString.cs b/src/runtime/InternString.cs index b6d9a0e4a..decb3981d 100644 --- a/src/runtime/InternString.cs +++ b/src/runtime/InternString.cs @@ -42,7 +42,7 @@ public static void Initialize() Debug.Assert(name == op.As()); SetIntern(name, op); var field = type.GetField("f" + name, PyIdentifierFieldFlags)!; - field.SetValue(null, op.rawPtr); + field.SetValue(null, op.DangerousGetAddressOrNull()); } } @@ -76,7 +76,7 @@ public static bool TryGetInterned(BorrowedReference op, out string s) private static void SetIntern(string s, PyString op) { _string2interns.Add(s, op); - _intern2strings.Add(op.rawPtr, s); + _intern2strings.Add(op.Reference.DangerousGetAddress(), s); } } } diff --git a/src/runtime/PythonTypes/PyObject.cs b/src/runtime/PythonTypes/PyObject.cs index 3d48e22ed..ce86753eb 100644 --- a/src/runtime/PythonTypes/PyObject.cs +++ b/src/runtime/PythonTypes/PyObject.cs @@ -27,7 +27,7 @@ public partial class PyObject : DynamicObject, IDisposable, ISerializable public StackTrace Traceback { get; } = new StackTrace(1); #endif - protected internal IntPtr rawPtr = IntPtr.Zero; + protected IntPtr rawPtr = IntPtr.Zero; internal readonly int run = Runtime.GetRun(); internal BorrowedReference obj => new (rawPtr); @@ -252,6 +252,8 @@ internal void Leak() rawPtr = IntPtr.Zero; } + internal IntPtr DangerousGetAddressOrNull() => rawPtr; + internal void CheckRun() { if (run != Runtime.GetRun()) diff --git a/src/runtime/Types/ReflectedClrType.cs b/src/runtime/Types/ReflectedClrType.cs index bbda4364f..41d568ac4 100644 --- a/src/runtime/Types/ReflectedClrType.cs +++ b/src/runtime/Types/ReflectedClrType.cs @@ -117,6 +117,6 @@ static ReflectedClrType AllocateClass(Type clrType) return new ReflectedClrType(type.Steal()); } - public override bool Equals(PyObject? other) => other != null && rawPtr == other.rawPtr; + public override bool Equals(PyObject? other) => rawPtr == other?.DangerousGetAddressOrNull(); public override int GetHashCode() => rawPtr.GetHashCode(); } diff --git a/src/runtime/Util/PythonReferenceComparer.cs b/src/runtime/Util/PythonReferenceComparer.cs index dd78f912d..63c35df57 100644 --- a/src/runtime/Util/PythonReferenceComparer.cs +++ b/src/runtime/Util/PythonReferenceComparer.cs @@ -13,10 +13,10 @@ public sealed class PythonReferenceComparer : IEqualityComparer public static PythonReferenceComparer Instance { get; } = new PythonReferenceComparer(); public bool Equals(PyObject? x, PyObject? y) { - return x?.rawPtr == y?.rawPtr; + return x?.DangerousGetAddressOrNull() == y?.DangerousGetAddressOrNull(); } - public int GetHashCode(PyObject obj) => obj.rawPtr.GetHashCode(); + public int GetHashCode(PyObject obj) => obj.DangerousGetAddressOrNull().GetHashCode(); private PythonReferenceComparer() { } } From 32d15eb2432558117fdf49b8686efa608891b056 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Sun, 17 Jul 2022 09:09:05 +1000 Subject: [PATCH 12/17] docs: Fix a few typos There are small typos in: - pythonnet/__init__.py - tests/test_import.py Fixes: - Should read `splitted` rather than `splited`. - Should read `loaded` rather than `laoded`. Signed-off-by: Tim Gates --- pythonnet/__init__.py | 2 +- tests/test_import.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pythonnet/__init__.py b/pythonnet/__init__.py index 2cde01233..7780bd389 100644 --- a/pythonnet/__init__.py +++ b/pythonnet/__init__.py @@ -145,7 +145,7 @@ def load(runtime: Union[clr_loader.Runtime, str, None] = None, **params: str) -> def unload() -> None: - """Explicitly unload a laoded runtime and shut down Python.NET""" + """Explicitly unload a loaded runtime and shut down Python.NET""" global _RUNTIME, _LOADER_ASSEMBLY if _LOADER_ASSEMBLY is not None: diff --git a/tests/test_import.py b/tests/test_import.py index 25877be15..877eacd84 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -15,7 +15,7 @@ def test_relative_missing_import(): def test_import_all_on_second_time(): """Test import all attributes after a normal import without '*'. - Due to import * only allowed at module level, the test body splited + Due to import * only allowed at module level, the test body splitted to a module file.""" from . import importtest del sys.modules[importtest.__name__] From 9eaf35ff4531a697f6513e50ebd4a44947b33f68 Mon Sep 17 00:00:00 2001 From: Rolf Madsen Date: Fri, 2 Sep 2022 09:38:42 +0200 Subject: [PATCH 13/17] Added support for marking properties with python types. Added support for specifying attributes on property-methods (eg. get/set attributes). --- src/runtime/PythonTypes/PyObject.cs | 8 +-- .../PythonTypes/PythonTypeAttribute.cs | 27 ++++++++++ src/runtime/Resources/clr.py | 12 +++-- src/runtime/Types/ClassDerived.cs | 52 ++++++++++++++++--- 4 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 src/runtime/PythonTypes/PythonTypeAttribute.cs diff --git a/src/runtime/PythonTypes/PyObject.cs b/src/runtime/PythonTypes/PyObject.cs index ce86753eb..99289fd05 100644 --- a/src/runtime/PythonTypes/PyObject.cs +++ b/src/runtime/PythonTypes/PyObject.cs @@ -25,7 +25,7 @@ public partial class PyObject : DynamicObject, IDisposable, ISerializable /// Trace stack for PyObject's construction /// public StackTrace Traceback { get; } = new StackTrace(1); -#endif +#endif protected IntPtr rawPtr = IntPtr.Zero; internal readonly int run = Runtime.GetRun(); @@ -165,7 +165,7 @@ public static PyObject FromManagedObject(object ob) { if (!Converter.ToManaged(obj, t, out var result, true)) { - throw new InvalidCastException("cannot convert object to target type", + throw new InvalidCastException($"Cannot convert object to target type '{t}'.", PythonException.FetchCurrentOrNull(out _)); } return result; @@ -235,7 +235,7 @@ public void Dispose() { GC.SuppressFinalize(this); Dispose(true); - + } internal StolenReference Steal() @@ -1314,7 +1314,7 @@ private bool TryCompare(PyObject arg, int op, out object @out) } return true; } - + public override bool TryBinaryOperation(BinaryOperationBinder binder, object arg, out object? result) { using var _ = Py.GIL(); diff --git a/src/runtime/PythonTypes/PythonTypeAttribute.cs b/src/runtime/PythonTypes/PythonTypeAttribute.cs new file mode 100644 index 000000000..7dbc413a9 --- /dev/null +++ b/src/runtime/PythonTypes/PythonTypeAttribute.cs @@ -0,0 +1,27 @@ +using System; + +namespace Python.Runtime; + +/// +/// Marks a property type with a specific python type. Normally, properties has .NET types, but if the property has a python type, +/// that cannot be represented in the propert type info, so this attribute is used to mark the property with the corresponding python type. +/// +public class PythonTypeAttribute : Attribute +{ + /// Type name. + public string TypeName { get; } + + /// Importable module name. + public string Module { get; } + + /// + /// Creates a new instance of PythonTypeAttribute. + /// + /// + /// + public PythonTypeAttribute(string pyTypeModule, string pyTypeName) + { + TypeName = pyTypeName; + Module = pyTypeModule; + } +} diff --git a/src/runtime/Resources/clr.py b/src/runtime/Resources/clr.py index 0f412452d..9d50d967a 100644 --- a/src/runtime/Resources/clr.py +++ b/src/runtime/Resources/clr.py @@ -61,7 +61,7 @@ def add_attribute(self, *args, **kwargs): class property(object): - def __init__(self, type, default): + def __init__(self, type, default = None): import weakref self._clr_property_type_ = type self.default = default @@ -70,9 +70,14 @@ def __init__(self, type, default): self.fget = 1 self.fset = 1 def __get__(self, instance, owner): + if self.fget != 1: + return self.fget(instance) v = self.values.get(instance, self.default) return v def __set__(self, instance, value): + if self.fset != 1: + self.fset(instance,value) + return self.values[instance] = value def add_attribute(self, *args, **kwargs): lst = [] @@ -84,8 +89,9 @@ def add_attribute(self, *args, **kwargs): self._clr_attributes_.extend(lst) return self - def __call__(self, type, default): - self2 = self.__class__(self._clr_property_type_, type, default) + def __call__(self, func): + self2 = self.__class__(self._clr_property_type_, None) + self2.fget = func self2._clr_attributes_ = self._clr_attributes_ return self2 class clrmethod(object): diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 726e4e41f..ec7527925 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -145,6 +145,10 @@ static CustomAttributeBuilder AddAttribute(PyObject attr) // named arguments and named properties or fields the same way. var tp = new PyTuple(attr); Type attribute = (Type) tp[0].AsManagedObject(typeof(object)); + if (typeof(Attribute).IsAssignableFrom(attribute) == false) + { + throw new Exception("This type is not an attribute type."); + } var args = (PyObject[]) tp[1].AsManagedObject(typeof(PyObject[])); var dict = new PyDict(tp[2]); @@ -168,9 +172,10 @@ static CustomAttributeBuilder AddAttribute(PyObject attr) if (parameter.ParameterType.IsArray && null != parameter.GetCustomAttribute(typeof(ParamArrayAttribute))) { int cnt = args.Length - i; - var values = Array.CreateInstance(parameter.ParameterType.GetElementType(), cnt); + var elemType = parameter.ParameterType.GetElementType(); + var values = Array.CreateInstance(elemType, cnt); for(int j = 0; j < cnt; j++) - values.SetValue(args[i + j], j); + values.SetValue(args[i + j].AsManagedObject(typeof(object)), j); paramValues.Add(values); break; } @@ -750,28 +755,61 @@ private static void AddPythonProperty(string propertyName, PyObject func, TypeBu MethodAttributes.SpecialName; using var pyPropertyType = func.GetAttr("_clr_property_type_"); - var propertyType = pyPropertyType.AsManagedObject(typeof(Type)) as Type; + var pyNativeType = new PyType(pyPropertyType); + Converter.ToManaged(pyPropertyType, typeof(Type), out var result, false); + var propertyType = result as Type; + string pyTypeName = null; + string pyTypeModule = null; if (propertyType == null) { - throw new ArgumentException("_clr_property_type must be a CLR type"); + propertyType = typeof(PyObject); + pyTypeModule = pyNativeType.GetAttr("__module__").ToString(); + //throw new ArgumentException("_clr_property_type must be a CLR type"); + pyTypeName = pyNativeType.Name; } PropertyBuilder propertyBuilder = typeBuilder.DefineProperty(propertyName, PropertyAttributes.None, propertyType, null); - if (func.HasAttr("_clr_attributes_")) + if (func.HasAttr("_clr_attributes_")) + { + using (var attributes = new PyList(func.GetAttr("_clr_attributes_"))) + { + foreach (var attr in attributes) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + propertyBuilder.SetCustomAttribute(builder); + } + } + } + + if (pyTypeName != null) + { + var cb = new CustomAttributeBuilder(typeof(PythonTypeAttribute).GetConstructors().First(), + new object[] {pyTypeModule, pyTypeName}); + propertyBuilder.SetCustomAttribute(cb); + } + + { + // load attribute set on functions + foreach (var fname in new[]{ "fget", "fset" }) { - using (var attributes = new PyList(func.GetAttr("_clr_attributes_"))) + using var pyfget = func.GetAttr(fname); + if (methodAssoc.TryGetValue(pyfget, out var lst)) { - foreach (var attr in attributes) + foreach (var attr in lst) { var builder = AddAttribute(attr); if (builder == null) throw new Exception(); propertyBuilder.SetCustomAttribute(builder); } + + methodAssoc.Remove(func); } } + } if (func.HasAttr("fget")) { From 9ed94f6cedef014918d8bdf7886f57cb15d2002b Mon Sep 17 00:00:00 2001 From: Rolf Madsen Date: Fri, 2 Sep 2022 15:03:48 +0200 Subject: [PATCH 14/17] Fixed issue calling base-base class implementation of methods. --- src/python_tests_runner/PythonTestRunner.cs | 1 + src/runtime/Types/ClassDerived.cs | 33 +++++++++++---------- src/testing/subclasstest.cs | 18 ++++++++--- tests/test_subclass.py | 24 +++++++++++++++ 4 files changed, 57 insertions(+), 19 deletions(-) diff --git a/src/python_tests_runner/PythonTestRunner.cs b/src/python_tests_runner/PythonTestRunner.cs index 9562c6a8d..dcc76e9d9 100644 --- a/src/python_tests_runner/PythonTestRunner.cs +++ b/src/python_tests_runner/PythonTestRunner.cs @@ -42,6 +42,7 @@ static IEnumerable PythonTestCases() yield return new[] { "test_subclass", "test_class_with_attributes" }; yield return new[] { "test_subclass", "test_class_with_advanced_attribute" }; yield return new[] { "test_subclass", "test_more_subclasses" }; + yield return new[] { "test_subclass", "test_more_subclasses2" }; yield return new[] { "test_subclass", "abstract_test" }; } diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index ec7527925..1465bc035 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -522,23 +522,26 @@ private static void AddVirtualMethod(MethodInfo method, Type baseType, TypeBuild string? baseMethodName = null; if (!method.IsAbstract) { - baseMethodName = "_" + baseType.Name + "__" + method.Name; - MethodBuilder baseMethodBuilder = typeBuilder.DefineMethod(baseMethodName, - MethodAttributes.Public | - MethodAttributes.Final | - MethodAttributes.HideBySig, - method.ReturnType, - parameterTypes); - - // emit the assembly for calling the original method using call instead of callvirt - ILGenerator baseIl = baseMethodBuilder.GetILGenerator(); - baseIl.Emit(OpCodes.Ldarg_0); - for (var i = 0; i < parameters.Length; ++i) + baseMethodName = "_" + method.DeclaringType.Name + "__" + method.Name; + if (baseType.GetMethod(baseMethodName) == null) { - baseIl.Emit(OpCodes.Ldarg, i + 1); + MethodBuilder baseMethodBuilder = typeBuilder.DefineMethod(baseMethodName, + MethodAttributes.Public | + MethodAttributes.Final | + MethodAttributes.HideBySig, + method.ReturnType, + parameterTypes); + + // emit the assembly for calling the original method using call instead of callvirt + ILGenerator baseIl = baseMethodBuilder.GetILGenerator(); + baseIl.Emit(OpCodes.Ldarg_0); + for (var i = 0; i < parameters.Length; ++i) + { + baseIl.Emit(OpCodes.Ldarg, i + 1); + } + baseIl.Emit(OpCodes.Call, method); + baseIl.Emit(OpCodes.Ret); } - baseIl.Emit(OpCodes.Call, method); - baseIl.Emit(OpCodes.Ret); } // override the original method with a new one that dispatches to python diff --git a/src/testing/subclasstest.cs b/src/testing/subclasstest.cs index 943366499..9b6071c15 100644 --- a/src/testing/subclasstest.cs +++ b/src/testing/subclasstest.cs @@ -151,8 +151,17 @@ public TestAttributeAttribute(int x, int y, string z = "x") } } + public abstract class SimpleClassBase + { + private int counter; + public virtual int IncrementThing() + { + return counter++; + } - public class SimpleClass + } + + public abstract class SimpleClass : SimpleClassBase { public bool Initialized; @@ -160,10 +169,11 @@ public SimpleClass() { Initialized = true; } - private int counter = 0; - public virtual int IncrementThing() + + public int CallIncrementThing() { - return ++counter; + var x = IncrementThing(); + return x; } public static void TestObject(object obj) diff --git a/tests/test_subclass.py b/tests/test_subclass.py index 585947d56..cdce67786 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -396,6 +396,7 @@ def __init__(self): SimpleClass.Pause(); super().__init__() def IncrementThing(self): + super().IncrementThing() return 6; SimpleClass.TestOnType(SubClass0) SimpleClass.TestOnType(SubClass1) @@ -465,3 +466,26 @@ class Derived(BaseClass): import gc gc.collect() +def test_more_subclasses2(): + import clr + class SubClass50(SimpleClass): + def __init__(self): + super().__init__() + def IncrementThing(self): + return super().IncrementThing() + + @clr.attribute(DebuggerDisplay("X")) + + class SubClass51(SubClass50): + __namespace__ = "TestModule" + def __init__(self): + super().__init__() + + def IncrementThing(self): + return super().IncrementThing() + 10 + x = SubClass51() + print(x.CallIncrementThing()) + print(x.CallIncrementThing()) + print(x.CallIncrementThing()) + + From da146d9bf97288f349772811084367f6e88b1ef5 Mon Sep 17 00:00:00 2001 From: Rolf Madsen Date: Fri, 28 Oct 2022 14:03:25 +0200 Subject: [PATCH 15/17] Merged with Pythonnet 3.0 RC.6. --- src/runtime/TypeManager.cs | 16 +++++----------- src/runtime/Types/ClassDerived.cs | 6 +++--- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 1099d6430..9746bc0ce 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -416,17 +416,11 @@ internal static NewReference CreateSubType(BorrowedReference py_name, IEnumerabl // create the new managed type subclassing the base managed type var baseClass = py_base_type.FirstOrDefault(); - if (null == baseClass) - { - return ReflectedClrType.CreateSubclass(baseClass, interfaces, name, - ns: (string?)namespaceStr, - assembly: (string?)assembly, - dict: dictRef); - } - else - { - return Exceptions.RaiseTypeError("invalid base class, expected CLR class type"); - } + + return ReflectedClrType.CreateSubclass(baseClass, interfaces, name, + ns: (string?)namespaceStr, + assembly: (string?)assembly, + dict: dictRef); } internal static IntPtr WriteMethodDef(IntPtr mdef, IntPtr name, IntPtr func, PyMethodFlags flags, IntPtr doc) diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 1465bc035..e06053c97 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -625,8 +625,8 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde } using var pyReturnType = func.GetAttr("_clr_return_type_"); - usingvar attributes = new PyList(func.GetAttr("_clr_attributes_"))) - using (var pyArgTypes = func.GetAttr("_clr_arg_types_"); + using var attributes = new PyList(func.GetAttr("_clr_attributes_")); + using var pyArgTypes = func.GetAttr("_clr_arg_types_"); using var pyArgTypesIter = PyIter.GetIter(pyArgTypes); var returnType = pyReturnType.AsManagedObject(typeof(Type)) as Type; if (returnType == null) @@ -1160,7 +1160,7 @@ public static void InvokeCtor(IPythonDerivedType obj, string origCtorName, objec // This doesn't actually destroy the object, it just sets the reference to this object // to be a weak reference and it will be destroyed when the C# object is destroyed. - PyType cc = ClassManager.GetClass(obj.GetType()); + PyType cc = ReflectedClrType.GetOrCreate(obj.GetType()); var args2 = new PyObject[args.Length]; for (int i = 0; i < args.Length; i++) args2[i] = args[i].ToPython(); From b280b1ba41ab225240a8d984603c27cb32fd4c13 Mon Sep 17 00:00:00 2001 From: Rolf Madsen Date: Fri, 28 Oct 2022 14:03:53 +0200 Subject: [PATCH 16/17] Fixed bug related to converting number to a string --- src/runtime/PythonTypes/PyInt.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/runtime/PythonTypes/PyInt.cs b/src/runtime/PythonTypes/PyInt.cs index 6b3dbf210..278056345 100644 --- a/src/runtime/PythonTypes/PyInt.cs +++ b/src/runtime/PythonTypes/PyInt.cs @@ -228,7 +228,8 @@ public BigInteger ToBigInteger() public string ToString(string format, IFormatProvider formatProvider) { using var _ = Py.GIL(); - return ToBigInteger().ToString(format, formatProvider); + object val = Runtime.PyLong_AsLongLong(obj); + return val?.ToString() ?? ToBigInteger().ToString(format, formatProvider); } public override TypeCode GetTypeCode() => TypeCode.Int64; From 61c5d999222f86345ae126e26310965bc6a2bc2c Mon Sep 17 00:00:00 2001 From: Rolf Madsen Date: Fri, 28 Oct 2022 14:05:05 +0200 Subject: [PATCH 17/17] Added support for Python 3.11. --- src/runtime/Native/TypeOffset311.cs | 143 ++++++++++++++++++++++++++++ src/runtime/TypeManager.cs | 23 +++-- src/runtime/Types/ManagedType.cs | 6 +- 3 files changed, 160 insertions(+), 12 deletions(-) create mode 100644 src/runtime/Native/TypeOffset311.cs diff --git a/src/runtime/Native/TypeOffset311.cs b/src/runtime/Native/TypeOffset311.cs new file mode 100644 index 000000000..bb10811ba --- /dev/null +++ b/src/runtime/Native/TypeOffset311.cs @@ -0,0 +1,143 @@ +// Auto-generated by geninterop.py. +// DO NOT MODIFY BY HAND. + +// Python 3.11: ABI flags: '' + +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + + [StructLayout(LayoutKind.Sequential)] + internal class TypeOffset311 : GeneratedTypeOffsets, ITypeOffsets + { + public TypeOffset311() + { + + } + // Auto-generated from PyHeapTypeObject in Python.h + public int ob_refcnt { get; private set; } + public int ob_type { get; private set; } + public int ob_size { get; private set; } + public int tp_name { get; private set; } + public int tp_basicsize { get; private set; } + public int tp_itemsize { get; private set; } + public int tp_dealloc { get; private set; } + public int tp_vectorcall_offset { get; private set; } + public int tp_getattr { get; private set; } + public int tp_setattr { get; private set; } + public int tp_as_async { get; private set; } + public int tp_repr { get; private set; } + public int tp_as_number { get; private set; } + public int tp_as_sequence { get; private set; } + public int tp_as_mapping { get; private set; } + public int tp_hash { get; private set; } + public int tp_call { get; private set; } + public int tp_str { get; private set; } + public int tp_getattro { get; private set; } + public int tp_setattro { get; private set; } + public int tp_as_buffer { get; private set; } + public int tp_flags { get; private set; } + public int tp_doc { get; private set; } + public int tp_traverse { get; private set; } + public int tp_clear { get; private set; } + public int tp_richcompare { get; private set; } + public int tp_weaklistoffset { get; private set; } + public int tp_iter { get; private set; } + public int tp_iternext { get; private set; } + public int tp_methods { get; private set; } + public int tp_members { get; private set; } + public int tp_getset { get; private set; } + public int tp_base { get; private set; } + public int tp_dict { get; private set; } + public int tp_descr_get { get; private set; } + public int tp_descr_set { get; private set; } + public int tp_dictoffset { get; private set; } + public int tp_init { get; private set; } + public int tp_alloc { get; private set; } + public int tp_new { get; private set; } + public int tp_free { get; private set; } + public int tp_is_gc { get; private set; } + public int tp_bases { get; private set; } + public int tp_mro { get; private set; } + public int tp_cache { get; private set; } + public int tp_subclasses { get; private set; } + public int tp_weaklist { get; private set; } + public int tp_del { get; private set; } + public int tp_version_tag { get; private set; } + public int tp_finalize { get; private set; } + public int tp_vectorcall { get; private set; } + public int am_await { get; private set; } + public int am_aiter { get; private set; } + public int am_anext { get; private set; } + public int am_send { get; private set; } + public int nb_add { get; private set; } + public int nb_subtract { get; private set; } + public int nb_multiply { get; private set; } + public int nb_remainder { get; private set; } + public int nb_divmod { get; private set; } + public int nb_power { get; private set; } + public int nb_negative { get; private set; } + public int nb_positive { get; private set; } + public int nb_absolute { get; private set; } + public int nb_bool { get; private set; } + public int nb_invert { get; private set; } + public int nb_lshift { get; private set; } + public int nb_rshift { get; private set; } + public int nb_and { get; private set; } + public int nb_xor { get; private set; } + public int nb_or { get; private set; } + public int nb_int { get; private set; } + public int nb_reserved { get; private set; } + public int nb_float { get; private set; } + public int nb_inplace_add { get; private set; } + public int nb_inplace_subtract { get; private set; } + public int nb_inplace_multiply { get; private set; } + public int nb_inplace_remainder { get; private set; } + public int nb_inplace_power { get; private set; } + public int nb_inplace_lshift { get; private set; } + public int nb_inplace_rshift { get; private set; } + public int nb_inplace_and { get; private set; } + public int nb_inplace_xor { get; private set; } + public int nb_inplace_or { get; private set; } + public int nb_floor_divide { get; private set; } + public int nb_true_divide { get; private set; } + public int nb_inplace_floor_divide { get; private set; } + public int nb_inplace_true_divide { get; private set; } + public int nb_index { get; private set; } + public int nb_matrix_multiply { get; private set; } + public int nb_inplace_matrix_multiply { get; private set; } + public int mp_length { get; private set; } + public int mp_subscript { get; private set; } + public int mp_ass_subscript { get; private set; } + public int sq_length { get; private set; } + public int sq_concat { get; private set; } + public int sq_repeat { get; private set; } + public int sq_item { get; private set; } + public int was_sq_slice { get; private set; } + public int sq_ass_item { get; private set; } + public int was_sq_ass_slice { get; private set; } + public int sq_contains { get; private set; } + public int sq_inplace_concat { get; private set; } + public int sq_inplace_repeat { get; private set; } + public int bf_getbuffer { get; private set; } + public int bf_releasebuffer { get; private set; } + public int name { get; private set; } + public int ht_slots { get; private set; } + public int qualname { get; private set; } + public int ht_cached_keys { get; private set; } + public int ht_module { get; private set; } + public int _ht_tpname { get; private set; } + public int spec_cache_getitem { get; private set; } + } +} diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 9746bc0ce..e7baa70e0 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -470,17 +470,20 @@ internal static PyType CreateMetatypeWithGCHandleOffset() int size = Util.ReadInt32(Runtime.PyTypeType, TypeOffset.tp_basicsize) + IntPtr.Size // tp_clr_inst_offset ; - var result = new PyType(new TypeSpec("clr._internal.GCOffsetBase", basicSize: size, - new TypeSpec.Slot[] - { - - }, - TypeFlags.Default | TypeFlags.HeapType | TypeFlags.HaveGC), - bases: new PyTuple(new[] { py_type })); - SetRequiredSlots(result, seen: new HashSet()); - - Runtime.PyType_Modified(result); + var slots = new[] { + new TypeSpec.Slot(TypeSlotID.tp_traverse, subtype_traverse), + new TypeSpec.Slot(TypeSlotID.tp_clear, subtype_clear) + }; + var result = new PyType( + new TypeSpec( + "clr._internal.GCOffsetBase", + basicSize: size, + slots: slots, + TypeFlags.Default | TypeFlags.HeapType | TypeFlags.HaveGC + ), + bases: new PyTuple(new[] { py_type }) + ); return result; } diff --git a/src/runtime/Types/ManagedType.cs b/src/runtime/Types/ManagedType.cs index 2ed9d7970..7d2d98ec9 100644 --- a/src/runtime/Types/ManagedType.cs +++ b/src/runtime/Types/ManagedType.cs @@ -148,8 +148,10 @@ protected static void ClearObjectDict(BorrowedReference ob) { BorrowedReference type = Runtime.PyObject_TYPE(ob); int instanceDictOffset = Util.ReadInt32(type, TypeOffset.tp_dictoffset); - Debug.Assert(instanceDictOffset > 0); - Runtime.Py_CLEAR(ob, instanceDictOffset); + //Debug.Assert(instanceDictOffset > 0); + // Python 3.11, sometimes this dict is less than zero. + if(instanceDictOffset > 0) + Runtime.Py_CLEAR(ob, instanceDictOffset); } protected static BorrowedReference GetObjectDict(BorrowedReference ob)