diff --git a/src/python_tests_runner/PythonTestRunner.cs b/src/python_tests_runner/PythonTestRunner.cs index f97cc5aec..085c73edf 100644 --- a/src/python_tests_runner/PythonTestRunner.cs +++ b/src/python_tests_runner/PythonTestRunner.cs @@ -17,6 +17,8 @@ public class PythonTestRunner [OneTimeSetUp] public void SetUp() { + Python.Runtime.Runtime.PythonDLL = + "C:\\Python37.2\\python37.dll"; PythonEngine.Initialize(); } @@ -36,6 +38,13 @@ static IEnumerable PythonTestCases() yield return new[] { "test_indexer", "test_boolean_indexer" }; yield return new[] { "test_delegate", "test_bool_delegate" }; yield return new[] { "test_subclass", "test_implement_interface_and_class" }; + yield return new[] { "test_subclass", "test_virtual_generic_method" }; + yield return new[] { "test_subclass", "test_interface_and_class_impl2" }; + yield return new[] { "test_subclass", "test_class_with_attributes" }; + yield return new[] { "test_subclass", "test_class_with_advanced_attribute" }; + yield return new[] { "test_subclass", "test_more_subclasses" }; + yield return new[] { "test_subclass", "test_more_subclasses2" }; + yield return new[] { "test_subclass", "abstract_test" }; } /// diff --git a/src/runtime/Native/TypeOffset311.cs b/src/runtime/Native/TypeOffset311.cs index de5afacb9..eaab68ce5 100644 --- a/src/runtime/Native/TypeOffset311.cs +++ b/src/runtime/Native/TypeOffset311.cs @@ -1,4 +1,3 @@ - // Auto-generated by geninterop.py. // DO NOT MODIFY BY HAND. diff --git a/src/runtime/PythonTypes/PyInt.cs b/src/runtime/PythonTypes/PyInt.cs index 0d00f5a13..df7c782e3 100644 --- a/src/runtime/PythonTypes/PyInt.cs +++ b/src/runtime/PythonTypes/PyInt.cs @@ -228,7 +228,8 @@ public BigInteger ToBigInteger() public string ToString(string format, IFormatProvider formatProvider) { using var _ = Py.GIL(); - return ToBigInteger().ToString(format, formatProvider); + object val = Runtime.PyLong_AsLongLong(obj); + return val?.ToString() ?? ToBigInteger().ToString(format, formatProvider); } public override TypeCode GetTypeCode() => TypeCode.Int64; diff --git a/src/runtime/PythonTypes/PyObject.cs b/src/runtime/PythonTypes/PyObject.cs index cf0c2a03f..de11eb6b7 100644 --- a/src/runtime/PythonTypes/PyObject.cs +++ b/src/runtime/PythonTypes/PyObject.cs @@ -25,7 +25,7 @@ public partial class PyObject : DynamicObject, IDisposable, ISerializable /// Trace stack for PyObject's construction /// public StackTrace Traceback { get; } = new StackTrace(1); -#endif +#endif protected IntPtr rawPtr = IntPtr.Zero; internal readonly int run = Runtime.GetRun(); @@ -165,7 +165,7 @@ public static PyObject FromManagedObject(object ob) { if (!Converter.ToManaged(obj, t, out var result, true)) { - throw new InvalidCastException("cannot convert object to target type", + throw new InvalidCastException($"Cannot convert object to target type '{t}'.", PythonException.FetchCurrentOrNull(out _)); } return result; @@ -235,7 +235,7 @@ public void Dispose() { GC.SuppressFinalize(this); Dispose(true); - + } internal StolenReference Steal() diff --git a/src/runtime/PythonTypes/PythonTypeAttribute.cs b/src/runtime/PythonTypes/PythonTypeAttribute.cs new file mode 100644 index 000000000..7dbc413a9 --- /dev/null +++ b/src/runtime/PythonTypes/PythonTypeAttribute.cs @@ -0,0 +1,27 @@ +using System; + +namespace Python.Runtime; + +/// +/// Marks a property type with a specific python type. Normally, properties has .NET types, but if the property has a python type, +/// that cannot be represented in the propert type info, so this attribute is used to mark the property with the corresponding python type. +/// +public class PythonTypeAttribute : Attribute +{ + /// Type name. + public string TypeName { get; } + + /// Importable module name. + public string Module { get; } + + /// + /// Creates a new instance of PythonTypeAttribute. + /// + /// + /// + public PythonTypeAttribute(string pyTypeModule, string pyTypeName) + { + TypeName = pyTypeName; + Module = pyTypeModule; + } +} diff --git a/src/runtime/Resources/clr.py b/src/runtime/Resources/clr.py index d4330a4d5..9d50d967a 100644 --- a/src/runtime/Resources/clr.py +++ b/src/runtime/Resources/clr.py @@ -21,16 +21,18 @@ def test(self): string z = x.test; // calls into python and returns "x" """ - def __init__(self, type_, fget=None, fset=None): + def __init__(self, type_, fget=None, fset=None, attributes = []): self.__name__ = getattr(fget, "__name__", None) self._clr_property_type_ = type_ self.fget = fget self.fset = fset - + self._clr_attributes_ = attributes def __call__(self, fget): - return self.__class__(self._clr_property_type_, + self.__class__(self._clr_property_type_, fget=fget, - fset=self.fset) + fset=self.fset, + attributes = self._clr_attributes_) + def setter(self, fset): self.fset = fset @@ -47,8 +49,51 @@ def __set__(self, instance, value): if not self.fset: raise AttributeError("%s is read-only" % self.__name__) return self.fset.__get__(instance, None)(value) + def add_attribute(self, *args, **kwargs): + lst = [] + if len(args) > 0: + if isinstance(args[0], tuple): + lst = args + else: + lst = [(args[0], args[1:], kwargs)] + self._clr_attributes_.extend(lst) + return self +class property(object): + def __init__(self, type, default = None): + import weakref + self._clr_property_type_ = type + self.default = default + self.values = weakref.WeakKeyDictionary() + self._clr_attributes_ = [] + self.fget = 1 + self.fset = 1 + def __get__(self, instance, owner): + if self.fget != 1: + return self.fget(instance) + v = self.values.get(instance, self.default) + return v + def __set__(self, instance, value): + if self.fset != 1: + self.fset(instance,value) + return + self.values[instance] = value + def add_attribute(self, *args, **kwargs): + lst = [] + if len(args) > 0: + if isinstance(args[0], tuple): + lst = args + else: + lst = [(args[0], args[1:], kwargs)] + self._clr_attributes_.extend(lst) + return self + + def __call__(self, func): + self2 = self.__class__(self._clr_property_type_, None) + self2.fget = func + self2._clr_attributes_ = self._clr_attributes_ + return self2 class clrmethod(object): """ Method decorator for exposing python methods to .NET. @@ -67,18 +112,59 @@ def test(self, x): int z = x.test("hello"); // calls into python and returns len("hello") """ - def __init__(self, return_type, arg_types, clrname=None, func=None): + def __init__(self, return_type = None, arg_types = [], clrname=None, func=None, **kwargs): + if return_type == None: + import System + return_type = System.Void self.__name__ = getattr(func, "__name__", None) self._clr_return_type_ = return_type self._clr_arg_types_ = arg_types self._clr_method_name_ = clrname or self.__name__ self.__func = func + if 'attributes' in kwargs: + self._clr_attributes_ = kwargs["attributes"] + else: + self._clr_attributes_ = [] def __call__(self, func): - return self.__class__(self._clr_return_type_, + self2 = self.__class__(self._clr_return_type_, self._clr_arg_types_, clrname=self._clr_method_name_, func=func) + self2._clr_attributes_ = self._clr_attributes_ + return self2 def __get__(self, instance, owner): return self.__func.__get__(instance, owner) + + def add_attribute(self, *args, **kwargs): + lst = [] + if len(args) > 0: + if isinstance(args[0], tuple): + lst = args + else: + lst = [(args[0], args[1:], kwargs)] + self._clr_attributes_.extend(lst) + return self + +class attribute(object): + + def __init__(self, *args, **kwargs): + lst = [] + if len(args) > 0: + if isinstance(args[0], tuple): + lst = args + else: + lst = [(args[0], args[1:], kwargs)] + import Python.Runtime + #todo: ensure that attributes only are pushed when @ is used. + self.attr = lst + for item in lst: + Python.Runtime.PythonDerivedType.PushAttribute(item) + + def __call__(self, x): + import Python.Runtime + for item in self.attr: + if Python.Runtime.PythonDerivedType.AssocAttribute(item, x): + pass + return x diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index a26ad67a9..a6fb3dae6 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -1794,6 +1794,12 @@ internal static void SetNoSiteFlag() return *Delegates.Py_NoSiteFlag; }); } + + internal static uint PyTuple_GetSize(BorrowedReference tuple) + { + IntPtr r = Delegates.PyTuple_Size(tuple); + return (uint)r.ToInt32(); + } } internal class BadPythonDllException : MissingMethodException diff --git a/src/runtime/StateSerialization/MaybeType.cs b/src/runtime/StateSerialization/MaybeType.cs index 884b7edb0..549c5f29e 100644 --- a/src/runtime/StateSerialization/MaybeType.cs +++ b/src/runtime/StateSerialization/MaybeType.cs @@ -15,7 +15,6 @@ internal struct MaybeType : ISerializable const string SerializationName = "n"; readonly string name; readonly Type type; - public string DeletedMessage { get @@ -38,6 +37,7 @@ public Type Value public string Name => name; public bool Valid => type != null; + public Type ValueOrNull => type; public override string ToString() { diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 592eefd55..9f8d662e0 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -137,6 +137,117 @@ internal static NewReference ToPython(IPythonDerivedType obj) return result; } + static CustomAttributeBuilder AddAttribute(PyObject attr) + { + // this is a bit complicated because we want to support both unnamed and named arguments + // in C# there is a discrepancy between property setters and named argument wrt attributes + // in python there is no difference. But luckily there is rarely a conflict, so we can treat + // named arguments and named properties or fields the same way. + var tp = new PyTuple(attr); + Type attribute = (Type) tp[0].AsManagedObject(typeof(object)); + if (typeof(Attribute).IsAssignableFrom(attribute) == false) + { + throw new Exception("This type is not an attribute type."); + } + + var args = (PyObject[]) tp[1].AsManagedObject(typeof(PyObject[])); + var dict = new PyDict(tp[2]); + var dict2 = new Dictionary(); + foreach (var key in dict.Keys()) + { + var k = key.As(); + dict2[k] = dict[key]; + } + + // todo support kwargs and tupleargs + var allconstructors = attribute.GetConstructors(); + foreach (var constructor in allconstructors) + { + var parameters = constructor.GetParameters(); + List paramValues = new List(); + HashSet accountedFor = new HashSet(); + for (int i = 0; i < parameters.Length; i++) + { + var parameter = parameters[i]; + if (parameter.ParameterType.IsArray && null != parameter.GetCustomAttribute(typeof(ParamArrayAttribute))) + { + int cnt = args.Length - i; + var elemType = parameter.ParameterType.GetElementType(); + var values = Array.CreateInstance(elemType, cnt); + for(int j = 0; j < cnt; j++) + values.SetValue(args[i + j].AsManagedObject(typeof(object)), j); + paramValues.Add(values); + break; + } + + PyObject argvalue = null; + if (args.Length <= i && dict2.TryGetValue(parameter.Name, out argvalue)) + { + accountedFor.Add(parameter.Name); + }else if (args.Length <= i && parameter.IsOptional) + { + paramValues.Add(parameter.DefaultValue); + continue; + }else if (args.Length <= i) + { + goto next; + } + else + { + argvalue = args[i]; + } + + var argval = argvalue.AsManagedObject(parameter.ParameterType); + if (parameter.ParameterType.IsAssignableFrom(argval?.GetType() ?? typeof(object))) + paramValues.Add(argval); + else if (parameter.IsOptional) + { + paramValues.Add(parameter.DefaultValue); + } + } + + List namedProperties = new List(); + List namedPropertyValues = new List(); + List namedFields = new List(); + List namedFieldValues = new List(); + + foreach (var key in dict2.Keys.Where(x => accountedFor.Contains(x) == false)) + { + var member = attribute.GetMember(key).FirstOrDefault(); + if (member == null) + goto next; + if (member is PropertyInfo prop) + { + namedProperties.Add(prop); + var argval = dict2[key].AsManagedObject(prop.PropertyType); + if (prop.PropertyType.IsAssignableFrom(argval?.GetType() ?? typeof(object))) + namedPropertyValues.Add(argval); + else + goto next; + } + if (member is FieldInfo field) + { + namedFields.Add(field); + var argval = dict2[key].AsManagedObject(field.FieldType); + if (field.FieldType.IsAssignableFrom(argval?.GetType() ?? typeof(object))) + namedFieldValues.Add(argval); + else + goto next; + } + } + + var cb = new CustomAttributeBuilder(constructor, paramValues.ToArray(), + namedProperties.ToArray(), namedPropertyValues.ToArray(), + namedFields.ToArray(), namedFieldValues.ToArray()); + return cb; + next: ; + } + + return null; + } + + + /// /// Creates a new managed type derived from a base type with any virtual /// methods overridden to call out to python if the associated python @@ -165,9 +276,7 @@ internal static Type CreateDerivedType(string name, Type baseClass = baseType; var interfaces = new HashSet { typeof(IPythonDerivedType) }; - foreach(var interfaceType in typeInterfaces) - interfaces.Add(interfaceType); - + foreach(var t in interfaces) interfaces.Add(t); // if the base type is an interface then use System.Object as the base class // and add the base type to the list of interfaces this new class will implement. if (baseType.IsInterface) @@ -176,26 +285,42 @@ internal static Type CreateDerivedType(string name, baseClass = typeof(object); } + bool isAbstract = false; + if (py_dict != null && Runtime.PyDict_Check(py_dict)) + { + using var dict = new PyDict(py_dict); + if (dict.HasKey("__clr_abstract__")) + isAbstract = true; + } + TypeBuilder typeBuilder = moduleBuilder.DefineType(name, - TypeAttributes.Public | TypeAttributes.Class, + TypeAttributes.Public | TypeAttributes.Class | (isAbstract ? TypeAttributes.Abstract : 0), baseClass, interfaces.ToArray()); // add a field for storing the python object pointer // FIXME: fb not used - FieldBuilder fb = typeBuilder.DefineField(PyObjName, + if (baseClass.GetField(PyObjName, PyObjFlags) == null) + { + FieldBuilder fb = typeBuilder.DefineField(PyObjName, #pragma warning disable CS0618 // Type or member is obsolete. OK for internal use. - typeof(UnsafeReferenceWithRun), + typeof(UnsafeReferenceWithRun), #pragma warning restore CS0618 // Type or member is obsolete - FieldAttributes.Private); + FieldAttributes.Public); + } // override any constructors - ConstructorInfo[] constructors = baseClass.GetConstructors(); + ConstructorInfo[] constructors = baseClass.GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic); foreach (ConstructorInfo ctor in constructors) { AddConstructor(ctor, baseType, typeBuilder); } + if (constructors.Length == 0) + { + AddConstructor(null, baseType, typeBuilder); + } + // Override any properties explicitly overridden in python var pyProperties = new HashSet(); if (py_dict != null && Runtime.PyDict_Check(py_dict)) @@ -250,6 +375,25 @@ internal static Type CreateDerivedType(string name, if (py_dict != null && Runtime.PyDict_Check(py_dict)) { using var dict = new PyDict(py_dict); + + if (dict.HasKey("__clr_attributes__")) + { + var attributes = new PyList(dict["__clr_attributes__"]); + foreach (var attr in attributes) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + typeBuilder.SetCustomAttribute(builder); + } + } + + foreach (var attr in PopAttributes()) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + typeBuilder.SetCustomAttribute(builder); + } + using var keys = dict.Keys(); foreach (PyObject pyKey in keys) { @@ -290,6 +434,7 @@ internal static Type CreateDerivedType(string name, Type type = typeBuilder.CreateType(); + // scan the assembly so the newly added class can be imported Assembly assembly = Assembly.GetAssembly(type); AssemblyManager.ScanAssembly(assembly); @@ -300,6 +445,8 @@ internal static Type CreateDerivedType(string name, return type; } + private static Dictionary pyTypeLookup = new Dictionary(); + /// /// Add a constructor override that calls the python ctor after calling the base type constructor. /// @@ -308,7 +455,7 @@ internal static Type CreateDerivedType(string name, /// TypeBuilder for the new type the ctor is to be added to private static void AddConstructor(ConstructorInfo ctor, Type baseType, TypeBuilder typeBuilder) { - ParameterInfo[] parameters = ctor.GetParameters(); + ParameterInfo[] parameters = ctor?.GetParameters() ?? Array.Empty(); Type[] parameterTypes = (from param in parameters select param.ParameterType).ToArray(); // create a method for calling the original constructor @@ -327,14 +474,15 @@ private static void AddConstructor(ConstructorInfo ctor, Type baseType, TypeBuil { il.Emit(OpCodes.Ldarg, i + 1); } - il.Emit(OpCodes.Call, ctor); + if(ctor != null) + il.Emit(OpCodes.Call, ctor); il.Emit(OpCodes.Ret); // override the original method with a new one that dispatches to python ConstructorBuilder cb = typeBuilder.DefineConstructor(MethodAttributes.Public | MethodAttributes.ReuseSlot | MethodAttributes.HideBySig, - ctor.CallingConvention, + ctor?.CallingConvention ?? CallingConventions.Any, parameterTypes); il = cb.GetILGenerator(); il.DeclareLocal(typeof(object[])); @@ -377,23 +525,26 @@ private static void AddVirtualMethod(MethodInfo method, Type baseType, TypeBuild string? baseMethodName = null; if (!method.IsAbstract) { - baseMethodName = "_" + baseType.Name + "__" + method.Name; - MethodBuilder baseMethodBuilder = typeBuilder.DefineMethod(baseMethodName, - MethodAttributes.Public | - MethodAttributes.Final | - MethodAttributes.HideBySig, - method.ReturnType, - parameterTypes); - - // emit the assembly for calling the original method using call instead of callvirt - ILGenerator baseIl = baseMethodBuilder.GetILGenerator(); - baseIl.Emit(OpCodes.Ldarg_0); - for (var i = 0; i < parameters.Length; ++i) + baseMethodName = "_" + method.DeclaringType.Name + "__" + method.Name; + if (baseType.GetMethod(baseMethodName) == null) { - baseIl.Emit(OpCodes.Ldarg, i + 1); + MethodBuilder baseMethodBuilder = typeBuilder.DefineMethod(baseMethodName, + MethodAttributes.Public | + MethodAttributes.Final | + MethodAttributes.HideBySig, + method.ReturnType, + parameterTypes); + + // emit the assembly for calling the original method using call instead of callvirt + ILGenerator baseIl = baseMethodBuilder.GetILGenerator(); + baseIl.Emit(OpCodes.Ldarg_0); + for (var i = 0; i < parameters.Length; ++i) + { + baseIl.Emit(OpCodes.Ldarg, i + 1); + } + baseIl.Emit(OpCodes.Call, method); + baseIl.Emit(OpCodes.Ret); } - baseIl.Emit(OpCodes.Call, method); - baseIl.Emit(OpCodes.Ret); } // override the original method with a new one that dispatches to python @@ -478,6 +629,7 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde } using var pyReturnType = func.GetAttr("_clr_return_type_"); + using var attributes = new PyList(func.GetAttr("_clr_attributes_")); using var pyArgTypes = func.GetAttr("_clr_arg_types_"); using var pyArgTypesIter = PyIter.GetIter(pyArgTypes); var returnType = pyReturnType.AsManagedObject(typeof(Type)) as Type; @@ -509,7 +661,27 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde returnType, argTypes.ToArray()); - ILGenerator il = methodBuilder.GetILGenerator(); + foreach (var attr in attributes) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + methodBuilder.SetCustomAttribute(builder); + } + + if (methodAssoc.TryGetValue(func, out var lst)) + { + foreach(var attr in lst) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + methodBuilder.SetCustomAttribute(builder); + } + + methodAssoc.Remove(func); + + } + + ILGenerator il = methodBuilder.GetILGenerator(); il.DeclareLocal(typeof(object[])); il.DeclareLocal(typeof(RuntimeMethodHandle)); @@ -596,16 +768,61 @@ private static void AddPythonProperty(string propertyName, PyObject func, TypeBu MethodAttributes.SpecialName; using var pyPropertyType = func.GetAttr("_clr_property_type_"); - var propertyType = pyPropertyType.AsManagedObject(typeof(Type)) as Type; + var pyNativeType = new PyType(pyPropertyType); + Converter.ToManaged(pyPropertyType, typeof(Type), out var result, false); + var propertyType = result as Type; + string pyTypeName = null; + string pyTypeModule = null; if (propertyType == null) { - throw new ArgumentException("_clr_property_type must be a CLR type"); + propertyType = typeof(PyObject); + pyTypeModule = pyNativeType.GetAttr("__module__").ToString(); + //throw new ArgumentException("_clr_property_type must be a CLR type"); + pyTypeName = pyNativeType.Name; } PropertyBuilder propertyBuilder = typeBuilder.DefineProperty(propertyName, PropertyAttributes.None, propertyType, null); + if (func.HasAttr("_clr_attributes_")) + { + using (var attributes = new PyList(func.GetAttr("_clr_attributes_"))) + { + foreach (var attr in attributes) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + propertyBuilder.SetCustomAttribute(builder); + } + } + } + + if (pyTypeName != null) + { + var cb = new CustomAttributeBuilder(typeof(PythonTypeAttribute).GetConstructors().First(), + new object[] {pyTypeModule, pyTypeName}); + propertyBuilder.SetCustomAttribute(cb); + } + + { + // load attribute set on functions + foreach (var fname in new[]{ "fget", "fset" }) + { + using var pyfget = func.GetAttr(fname); + if (methodAssoc.TryGetValue(pyfget, out var lst)) + { + foreach (var attr in lst) + { + var builder = AddAttribute(attr); + if (builder == null) throw new Exception(); + propertyBuilder.SetCustomAttribute(builder); + } + + methodAssoc.Remove(func); + } + } + } if (func.HasAttr("fget")) { @@ -701,8 +918,47 @@ private static ModuleBuilder GetModuleBuilder(string assemblyName, string module [Obsolete(Util.InternalUseOnly)] public class PythonDerivedType { + private static List attributesStack = new List(); + internal static Dictionary> methodAssoc = new Dictionary>(); + public static void PushAttribute(PyObject obj) + { + using var _ = Py.GIL(); + var tp = new PyTuple(obj); + attributesStack.Add(tp); + } + + public static bool AssocAttribute(PyObject obj, PyObject func) + { + using var _ = Py.GIL(); + var tp = new PyTuple(obj); + for (int i = 0; i < attributesStack.Count; i++) + { + + if (tp.BorrowNullable()== attributesStack[i].BorrowNullable()) + { + attributesStack.RemoveAt(i); + if (!methodAssoc.TryGetValue(func, out var lst)) + { + lst = methodAssoc[func] = new List(); + } + lst.Add(tp); + return true; + } + } + return false; + + } + + + public static IEnumerable PopAttributes() + { + if (attributesStack.Count == 0) return Array.Empty(); + var attrs = attributesStack; + attributesStack = new List(); + return attrs; + } internal const string PyObjName = "__pyobj__"; - internal const BindingFlags PyObjFlags = BindingFlags.Instance | BindingFlags.NonPublic; + internal const BindingFlags PyObjFlags = BindingFlags.Instance | BindingFlags.Public | BindingFlags.FlattenHierarchy; /// /// This is the implementation of the overridden methods in the derived @@ -907,6 +1163,7 @@ public static void InvokeSetProperty(IPythonDerivedType obj, string propertyN } } + static private PyObject pin; public static void InvokeCtor(IPythonDerivedType obj, string origCtorName, object[] args) { var selfRef = GetPyObj(obj); @@ -917,8 +1174,22 @@ public static void InvokeCtor(IPythonDerivedType obj, string origCtorName, objec // In the end we decrement the python object's reference count. // This doesn't actually destroy the object, it just sets the reference to this object // to be a weak reference and it will be destroyed when the C# object is destroyed. - using var self = CLRObject.GetReference(obj, obj.GetType()); - SetPyObj(obj, self.Borrow()); + + PyType cc = ReflectedClrType.GetOrCreate(obj.GetType()); + var args2 = new PyObject[args.Length]; + for (int i = 0; i < args.Length; i++) + args2[i] = args[i].ToPython(); + // create an instance of the class and steal the reference. + using (var obj2 = cc.Invoke(args2, null)) + { + // somehow if this is not done this object never gets a reference count + // and things breaks later. + // I am not sure what it does though. + GCHandle gc = GCHandle.Alloc(obj); + // hand over the reference. + var py = obj2.NewReferenceOrNull(); + SetPyObj(obj, py.Borrow()); + } } // call the base constructor diff --git a/src/runtime/Types/ManagedType.cs b/src/runtime/Types/ManagedType.cs index 97a19497c..7cb789b90 100644 --- a/src/runtime/Types/ManagedType.cs +++ b/src/runtime/Types/ManagedType.cs @@ -149,6 +149,7 @@ protected static void ClearObjectDict(BorrowedReference ob) BorrowedReference type = Runtime.PyObject_TYPE(ob); int instanceDictOffset = Util.ReadInt32(type, TypeOffset.tp_dictoffset); // Debug.Assert(instanceDictOffset > 0); + // Python 3.11, sometimes this dict is less than zero. if (instanceDictOffset > 0) Runtime.Py_CLEAR(ob, instanceDictOffset); } diff --git a/src/runtime/Types/MetaType.cs b/src/runtime/Types/MetaType.cs index 57fcaa232..2ee9ec6b1 100644 --- a/src/runtime/Types/MetaType.cs +++ b/src/runtime/Types/MetaType.cs @@ -169,15 +169,27 @@ public static NewReference tp_new(BorrowedReference tp, BorrowedReference args, // into python. if (null != dict) { + var btt = baseTypes.FirstOrDefault().type.ValueOrNull; + var ctor = btt?.GetConstructors(BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic) + .FirstOrDefault(x => x.GetParameters().Any() == false); using var clsDict = new PyDict(dict); - if (clsDict.HasKey("__assembly__") || clsDict.HasKey("__namespace__")) + + if (clsDict.HasKey("__assembly__") || clsDict.HasKey("__namespace__") + || (ctor != null)) { + if (!clsDict.HasKey("__namespace__")) + { + clsDict["__namespace__"] = + (clsDict["__module__"].ToString()).ToPython(); + } + return TypeManager.CreateSubType(name, baseTypes[0], interfaces, clsDict); + } + } var base_type = Runtime.PyTuple_GetItem(bases, 0); - // otherwise just create a basic type without reflecting back into the managed side. IntPtr func = Util.ReadIntPtr(Runtime.PyTypeType, TypeOffset.tp_new); NewReference type = NativeCall.Call_3(func, tp, args, kw); diff --git a/src/runtime/Types/ModuleObject.cs b/src/runtime/Types/ModuleObject.cs index f641b393e..73b8ff339 100644 --- a/src/runtime/Types/ModuleObject.cs +++ b/src/runtime/Types/ModuleObject.cs @@ -132,6 +132,23 @@ public NewReference GetAttribute(string name, bool guess) return new NewReference(c); } + // attribute names + // for attributes without the Attribute suffix, create an attribute builder. + var qname2 = qname + "Attribute"; + var type2 = AssemblyManager.LookupTypes(qname2).FirstOrDefault(t => t.IsPublic); + if (type2 != null) + { + var str = "def {2}attrbuilder(*args, **kwargs):\n" + + " import {1}\n" + + " return ({0}, args, kwargs)\n"; + str = string.Format(str, qname2, _namespace, name); + PythonEngine.Exec(str); + var obj = PythonEngine.Eval(name + "attrbuilder"); + var o = obj.NewReferenceOrNull(); + this.StoreAttribute(name, o.Borrow()); + return new NewReference(o); + } + // We didn't find the name, so we may need to see if there is a // generic type with this base name. If so, we'll go ahead and // return it. Note that we store the mapping of the unmangled @@ -322,7 +339,7 @@ public static NewReference tp_getattro(BorrowedReference ob, BorrowedReference k if (attr.IsNull()) { - Exceptions.SetError(Exceptions.AttributeError, name); + Exceptions.SetError(Exceptions.AttributeError, $"name '{name}' is not defined in module '{self.moduleName}'."); return default; } diff --git a/src/testing/classtest.cs b/src/testing/classtest.cs index 993afdfc9..d000e3770 100644 --- a/src/testing/classtest.cs +++ b/src/testing/classtest.cs @@ -60,15 +60,4 @@ public ClassCtorTest2(string v) internal class InternalClass { } - - public class SimpleClass - { - public static void TestObject(object obj) - { - if ((!(obj is ISayHello1 && obj is SimpleClass))) - { - throw new Exception("Expected ISayHello and SimpleClass instance"); - } - } - } } diff --git a/src/testing/generictest.cs b/src/testing/generictest.cs index b333910c2..a78fd5104 100644 --- a/src/testing/generictest.cs +++ b/src/testing/generictest.cs @@ -95,6 +95,11 @@ public string Overloaded(int arg1, int arg2, string arg3) { return arg3; } + + public virtual Q VirtualOverloaded(Q arg) + { + return arg; + } } public class GenericStaticMethodTest @@ -118,10 +123,7 @@ public static Q Overloaded(Q arg) return arg; } - public static U Overloaded(Q arg1, U arg2) - { - return arg2; - } + public static string Overloaded(int arg1, int arg2, string arg3) { diff --git a/src/testing/subclasstest.cs b/src/testing/subclasstest.cs index ab0b73368..9b6071c15 100644 --- a/src/testing/subclasstest.cs +++ b/src/testing/subclasstest.cs @@ -1,5 +1,8 @@ using System; using System.Collections.Generic; +using System.Threading; + +using Python.Runtime; namespace Python.Test { @@ -124,4 +127,134 @@ public static int test_event(IInterfaceTest x, int value) return et.value; } } + + public interface ISimpleInterface + { + bool Ok(); + } + public interface ISimpleInterface2 + { + int Execute(CancellationToken token); + } + public class TestAttributeAttribute: Attribute + { + public int X { get; set; } + public int Y { get; set; } + public string Z { get; set; } + public string W { get; set; } + public TestAttributeAttribute(int x, int y, string z = "x") + { + X = x; + Y = y; + Z = z; + + } + } + + public abstract class SimpleClassBase + { + private int counter; + public virtual int IncrementThing() + { + return counter++; + } + + } + + public abstract class SimpleClass : SimpleClassBase + { + public bool Initialized; + + public SimpleClass() + { + Initialized = true; + } + + public int CallIncrementThing() + { + var x = IncrementThing(); + return x; + } + + public static void TestObject(object obj) + { + if (obj is ISimpleInterface si) + { + if (!si.Ok()) + throw new Exception(); + + }else if (obj is ISimpleInterface2 si2) + { + si2.Execute(CancellationToken.None); + + } + else + { + throw new Exception(); + } + } + public static void TestObjectProperty(object obj, string prop, double newval) + { + obj.GetType().GetProperty(prop).SetValue(obj, newval); + var val = obj.GetType().GetProperty(prop).GetValue(obj); + if (!Equals(newval, val)) + throw new Exception(); + } + + private static SimpleClass objStore; + public static void Test1(SimpleClass obj) + { + objStore = obj; + int x = obj.IncrementThing(); + } + + public static void Test2() + { + GC.Collect(); + + var threads = new Thread[20]; + for(int i = 0; i < threads.Length; i++) + threads[i] = new Thread(() => TestObjectProperty(objStore, "X", 10.0)); + for (int i = 0; i < threads.Length; i++) + threads[i].Start(); + for (int i = 0; i < threads.Length; i++) + threads[i].Join(); + } + + public static object InvokeCtor(Type t) + { + var obj = Activator.CreateInstance(t); + return obj; + } + + public object TestObj { get; set; } + + public static object TestOnType(Type t) + { + using (Py.GIL()) + { + var obj = (SimpleClass) Activator.CreateInstance(t); + //obj = obj.ToPython().As(); + obj.TestObj = new object(); + var py = obj.ToPython(); + var man = py.As(); + if (!ReferenceEquals(man, obj)) + throw new Exception("Same object expected"); + var setObj = py.GetAttr("TestObj").As(); + if (setObj == null) + throw new NullReferenceException(); + if (ReferenceEquals(setObj, obj.TestObj) == false) + throw new Exception("!!"); + + + return obj; + } + } + + public static void Pause() + { + + } + + } } diff --git a/tests/test_subclass.py b/tests/test_subclass.py index c6ab7650f..3fa2d9c84 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -7,9 +7,13 @@ """Test sub-classing managed types""" import System +from System import (Console, Attribute, Double) +from System.Diagnostics import (DebuggerDisplay, DebuggerDisplayAttribute, Debug) +from System.ComponentModel import (Browsable, BrowsableAttribute) +from System.Threading import (CancellationToken) import pytest -from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, - FunctionsTest, IGenericInterface, GenericVirtualMethodTest, SimpleClass, ISayHello1) +from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, FunctionsTest,IGenericInterface, GenericVirtualMethodTest, ISimpleInterface, SimpleClass, TestAttribute, TestAttributeAttribute, ISimpleInterface2, ISayHello1) +import Python.Test from System.Collections.Generic import List @@ -275,6 +279,161 @@ class TestX(System.Object): t = TestX() assert t.q == 1 +def test_virtual_generic_method(): + class OverloadingSubclass(GenericVirtualMethodTest): + __namespace__ = "test_virtual_generic_method_cls" + class OverloadingSubclass2(OverloadingSubclass): + pass + obj = OverloadingSubclass() + assert obj.VirtMethod[int](5) == 5 + +def test_interface_and_class_impl(): + class OverloadingSubclass(GenericVirtualMethodTest): + __namespace__ = "test_virtual_generic_method_cls" + class OverloadingSubclass2(OverloadingSubclass): + pass + obj = OverloadingSubclass() + assert obj.VirtMethod[int](5) == 5 + +def test_interface_and_class_impl2(): + class DualSubClass(ISimpleInterface, SimpleClass): + def Ok(self): + return True + class DualSubClass2(ISimpleInterface): + def Ok(self): + return True + class DualSubClass3(ISimpleInterface2): + def Execute(self, cancellationToken): + return 0 + try: + class DualSubClass4(Python.Test.ISimpleInterface3): + def Execute(self, cancellationToken): + return 0 + assert False # An exception should be thrown. + except AttributeError as ae: + assert ("not defined" in str(ae)) + + obj = DualSubClass() + SimpleClass.TestObject(obj) + obj = DualSubClass2() + SimpleClass.TestObject(obj) + + obj2 = DualSubClass3(); + SimpleClass.TestObject(obj2) + #obj2.Execute(CancellationToken.None) + +def test_class_with_attributes(): + import clr + @clr.attribute(Browsable(False)) + class ClassWithAttributes(ISimpleInterface, SimpleClass): + __clr_attributes__ = [DebuggerDisplay("X: {X}")] + @clr.attribute(Browsable(True)) + def Ok(self): + return True + @clr.attribute(Browsable(True)) + @clr.clrmethod(int, [int]) + def Method1(x): + return x + + X = clr.property(Double, 1.0).add_attribute(DebuggerDisplay("Asd")) + obj = ClassWithAttributes() + tp = obj.GetType() + founddisplay = 0 + foundbrowsable = 0 + for attr in Attribute.GetCustomAttributes(tp): + if isinstance(attr, DebuggerDisplayAttribute): + founddisplay = founddisplay + 1 + if isinstance(attr, BrowsableAttribute): + foundbrowsable = foundbrowsable + 1 + SimpleClass.TestObject(obj) + found_display_on_property = 0 + for attr in Attribute.GetCustomAttributes(tp.GetProperty("X")): + if isinstance(attr, DebuggerDisplayAttribute): + found_display_on_property = found_display_on_property + 1 + found_display_on_method = 0 + for attr in Attribute.GetCustomAttributes(tp.GetMethod("Method1")): + if isinstance(attr, BrowsableAttribute): + found_display_on_method = found_display_on_method + 1 + assert founddisplay == 1 + assert found_display_on_property == 1 + assert found_display_on_method == 1 + assert foundbrowsable == 1 + assert obj.X == 1.0 + SimpleClass.TestObjectProperty(obj, "X", 10.0) +def test_class_with_advanced_attribute(): + import clr + @clr.attribute(TestAttribute(1, 2, z = "A", W = "B")) + class ClassWithAttributes2(ISimpleInterface, SimpleClass): + pass + @clr.attribute(TestAttributeAttribute, 1, 2, z = "A", W = "B") + class ClassWithAttributes3(ISimpleInterface, SimpleClass): + X = clr.property(Double, 1.0).add_attribute(TestAttributeAttribute, 1, 2) + + c = ClassWithAttributes2() + c2 = ClassWithAttributes3() + +def test_subclass_ctor(): + import clr + class SubClass0(SimpleClass): + pass + class SubClass1(SubClass0): + def __init__(self): + super().__init__() + class SubClass2(SubClass1): + __namespace__ = "TestModule" + def __init__(self): + super().__init__() + SimpleClass.TestOnType(SubClass0) + SimpleClass.TestOnType(SubClass1) + SimpleClass.TestOnType(SubClass2) + +def test_more_subclasses(): + import clr + class SubClass0(SimpleClass): + pass + class SubClass1(SubClass0): + X = clr.property(Double, 1.0) + def __init__(self): + super().__init__() + self.Y = 10.0 + SimpleClass.Pause(); + + @clr.attribute(DebuggerDisplay("X")) + + class SubClass2(SubClass1): + __namespace__ = "TestModule" + def __init__(self): + SimpleClass.Pause(); + super().__init__() + def IncrementThing(self): + super().IncrementThing() + return 6; + SimpleClass.TestOnType(SubClass0) + SimpleClass.TestOnType(SubClass1) + SimpleClass.TestOnType(SubClass2) + obj = SimpleClass.InvokeCtor(SubClass2) + + obj2 = SubClass2() + tp = obj.GetType() + obj.X = 5.0 + assert obj.Y == 10.0 + assert obj2.Y == 10.0 + assert obj.Initialized == True + assert obj2.Initialized == True + SimpleClass.Test1(obj) + obj = None + SimpleClass.Test2() + +def abstract_test(): + class abstractClass(SimpleClass): + __clr_abstract__ = True + failed = False + try: + abstractClass() + except: + failed = True + assert failed + def test_construction_from_clr(): import clr calls = [] @@ -317,6 +476,29 @@ class Derived(BaseClass): import gc gc.collect() +def test_more_subclasses2(): + import clr + class SubClass50(SimpleClass): + def __init__(self): + super().__init__() + def IncrementThing(self): + return super().IncrementThing() + + @clr.attribute(DebuggerDisplay("X")) + + class SubClass51(SubClass50): + __namespace__ = "TestModule" + def __init__(self): + super().__init__() + + def IncrementThing(self): + return super().IncrementThing() + 10 + x = SubClass51() + print(x.CallIncrementThing()) + print(x.CallIncrementThing()) + print(x.CallIncrementThing()) + + def test_generic_interface(): from System import Int32