From 65debff27e8dff68d6f36880331af77712bf3e83 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Sun, 21 Feb 2021 22:58:21 -0800 Subject: [PATCH 1/3] TypeFlags is an enum --- src/runtime/clrobject.cs | 2 +- src/runtime/interop.cs | 56 ++++++++++++++++++++------------------ src/runtime/managedtype.cs | 6 ++-- src/runtime/metatype.cs | 6 ++-- src/runtime/typemanager.cs | 16 +++++------ 5 files changed, 45 insertions(+), 41 deletions(-) diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 0a352b381..0aa829ee6 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -14,7 +14,7 @@ internal CLRObject(object ob, IntPtr tp) System.Diagnostics.Debug.Assert(tp != IntPtr.Zero); IntPtr py = Runtime.PyType_GenericAlloc(tp, 0); - long flags = Util.ReadCLong(tp, TypeOffset.tp_flags); + var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Subclass) != 0) { IntPtr dict = Marshal.ReadIntPtr(py, ObjectOffset.TypeDictOffset(tp)); diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 0f56f77d9..c5958e0f7 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -344,38 +344,42 @@ public static void FreeModuleDef(IntPtr ptr) /// Note that the two values reserved for stackless have been put /// to good use as PythonNet specific flags (Managed and Subclass) /// - internal class TypeFlags + // Py_TPFLAGS_* + [Flags] + public enum TypeFlags: int { - public const int HeapType = (1 << 9); - public const int BaseType = (1 << 10); - public const int Ready = (1 << 12); - public const int Readying = (1 << 13); - public const int HaveGC = (1 << 14); + HeapType = (1 << 9), + BaseType = (1 << 10), + Ready = (1 << 12), + Readying = (1 << 13), + HaveGC = (1 << 14), // 15 and 16 are reserved for stackless - public const int HaveStacklessExtension = 0; + HaveStacklessExtension = 0, /* XXX Reusing reserved constants */ - public const int Managed = (1 << 15); // PythonNet specific - public const int Subclass = (1 << 16); // PythonNet specific - public const int HaveIndex = (1 << 17); + /// PythonNet specific + Managed = (1 << 15), + /// PythonNet specific + Subclass = (1 << 16), + HaveIndex = (1 << 17), /* Objects support nb_index in PyNumberMethods */ - public const int HaveVersionTag = (1 << 18); - public const int ValidVersionTag = (1 << 19); - public const int IsAbstract = (1 << 20); - public const int HaveNewBuffer = (1 << 21); + HaveVersionTag = (1 << 18), + ValidVersionTag = (1 << 19), + IsAbstract = (1 << 20), + HaveNewBuffer = (1 << 21), // TODO: Implement FastSubclass functions - public const int IntSubclass = (1 << 23); - public const int LongSubclass = (1 << 24); - public const int ListSubclass = (1 << 25); - public const int TupleSubclass = (1 << 26); - public const int StringSubclass = (1 << 27); - public const int UnicodeSubclass = (1 << 28); - public const int DictSubclass = (1 << 29); - public const int BaseExceptionSubclass = (1 << 30); - public const int TypeSubclass = (1 << 31); - - public const int Default = ( + IntSubclass = (1 << 23), + LongSubclass = (1 << 24), + ListSubclass = (1 << 25), + TupleSubclass = (1 << 26), + StringSubclass = (1 << 27), + UnicodeSubclass = (1 << 28), + DictSubclass = (1 << 29), + BaseExceptionSubclass = (1 << 30), + TypeSubclass = (1 << 31), + + Default = ( HaveStacklessExtension | - HaveVersionTag); + HaveVersionTag), } diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 09e8a3be2..d3ee697fd 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -92,7 +92,7 @@ internal static ManagedType GetManagedObject(IntPtr ob) tp = ob; } - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); + var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Managed) != 0) { IntPtr op = tp == ob @@ -117,7 +117,7 @@ internal static ManagedType GetManagedObjectType(IntPtr ob) if (ob != IntPtr.Zero) { IntPtr tp = Runtime.PyObject_TYPE(ob); - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); + var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Managed) != 0) { tp = Marshal.ReadIntPtr(tp, TypeOffset.magic()); @@ -152,7 +152,7 @@ internal static bool IsManagedType(IntPtr ob) tp = ob; } - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); + var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Managed) != 0) { return true; diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 36b406c7b..68dae2508 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -147,13 +147,13 @@ public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) return IntPtr.Zero; } - int flags = TypeFlags.Default; + var flags = TypeFlags.Default; flags |= TypeFlags.Managed; flags |= TypeFlags.HeapType; flags |= TypeFlags.BaseType; flags |= TypeFlags.Subclass; flags |= TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); + Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); TypeManager.CopySlot(base_type, type, TypeOffset.tp_dealloc); @@ -285,7 +285,7 @@ public static void tp_dealloc(IntPtr tp) { // Fix this when we dont cheat on the handle for subclasses! - var flags = Util.ReadCLong(tp, TypeOffset.tp_flags); + var flags = (TypeFlags)Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Subclass) == 0) { IntPtr gc = Marshal.ReadIntPtr(tp, TypeOffset.magic()); diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index e0564b243..01aceb656 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -171,9 +171,9 @@ internal static IntPtr CreateType(Type impl) SlotsHolder slotsHolder = CreateSolotsHolder(type); InitializeSlots(type, impl, slotsHolder); - int flags = TypeFlags.Default | TypeFlags.Managed | + var flags = TypeFlags.Default | TypeFlags.Managed | TypeFlags.HeapType | TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); + Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); if (Runtime.PyType_Ready(type) != 0) { @@ -286,12 +286,12 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Runtime.XIncref(base_); } - const int flags = TypeFlags.Default + const TypeFlags flags = TypeFlags.Default | TypeFlags.Managed | TypeFlags.HeapType | TypeFlags.BaseType | TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); + Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); OperatorMethod.FixupSlots(type, clrType); // Leverage followup initialization from the Python runtime. Note @@ -457,11 +457,11 @@ internal static IntPtr CreateMetaType(Type impl, out SlotsHolder slotsHolder) int size = TypeOffset.magic() + IntPtr.Size; Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, new IntPtr(size)); - const int flags = TypeFlags.Default + const TypeFlags flags = TypeFlags.Default | TypeFlags.Managed | TypeFlags.HeapType | TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); + Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); // Slots will inherit from TypeType, it's not neccesary for setting them. // Inheried slots: @@ -562,11 +562,11 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl) Marshal.WriteIntPtr(type, TypeOffset.tp_base, base_); Runtime.XIncref(base_); - int flags = TypeFlags.Default; + var flags = TypeFlags.Default; flags |= TypeFlags.Managed; flags |= TypeFlags.HeapType; flags |= TypeFlags.HaveGC; - Util.WriteCLong(type, TypeOffset.tp_flags, flags); + Util.WriteCLong(type, TypeOffset.tp_flags, (int)flags); CopySlot(base_, type, TypeOffset.tp_traverse); CopySlot(base_, type, TypeOffset.tp_clear); From 6312330e39ed01d5d8773e7ea71a2e0aa12b257f Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Sun, 21 Feb 2021 22:58:38 -0800 Subject: [PATCH 2/3] use C# 9 everywhere by default --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index edc8ba513..e0cd93ede 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -4,7 +4,7 @@ Copyright (c) 2006-2020 The Contributors of the Python.NET Project pythonnet Python.NET - 7.3 + 9.0 false From d3c565475cee7c5ed13b59f79d7865c8e0c20d55 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Sun, 21 Feb 2021 23:02:52 -0800 Subject: [PATCH 3/3] added new class PyType to wrap Python type objects and enable new type construction from PyType_Spec (TypeSpec class) --- CHANGELOG.md | 1 + src/embed_tests/TestPyType.cs | 45 ++++++++++ src/runtime/TypeSpec.cs | 121 +++++++++++++++++++++++++++ src/runtime/native/NativeTypeSpec.cs | 45 ++++++++++ src/runtime/pytype.cs | 50 +++++++++++ src/runtime/runtime.cs | 4 + 6 files changed, 266 insertions(+) create mode 100644 src/embed_tests/TestPyType.cs create mode 100644 src/runtime/TypeSpec.cs create mode 100644 src/runtime/native/NativeTypeSpec.cs create mode 100644 src/runtime/pytype.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7fbd6f7b4..0d9a79d21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). - Add GetPythonThreadID and Interrupt methods in PythonEngine - Ability to implement delegates with `ref` and `out` parameters in Python, by returning the modified parameter values in a tuple. ([#1355][i1355]) +- `PyType` - a wrapper for Python type objects, that also permits creating new heap types from `TypeSpec` ### Changed - Drop support for Python 2, 3.4, and 3.5 diff --git a/src/embed_tests/TestPyType.cs b/src/embed_tests/TestPyType.cs new file mode 100644 index 000000000..02142b782 --- /dev/null +++ b/src/embed_tests/TestPyType.cs @@ -0,0 +1,45 @@ +using System.Text; + +using NUnit.Framework; + +using Python.Runtime; +using Python.Runtime.Native; + +namespace Python.EmbeddingTest +{ + public class TestPyType + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void CanCreateHeapType() + { + const string name = "nÁmæ"; + const string docStr = "dÁcæ"; + + using var doc = new StrPtr(docStr, Encoding.UTF8); + var spec = new TypeSpec( + name: name, + basicSize: ObjectOffset.Size(Runtime.Runtime.PyTypeType), + slots: new TypeSpec.Slot[] { + new (TypeSlotID.tp_doc, doc.RawPointer), + }, + TypeFlags.Default | TypeFlags.HeapType + ); + + using var type = new PyType(spec); + Assert.AreEqual(name, type.GetAttr("__name__").As()); + Assert.AreEqual(docStr, type.GetAttr("__doc__").As()); + } + } +} diff --git a/src/runtime/TypeSpec.cs b/src/runtime/TypeSpec.cs new file mode 100644 index 000000000..87c0f94bc --- /dev/null +++ b/src/runtime/TypeSpec.cs @@ -0,0 +1,121 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.InteropServices; + +namespace Python.Runtime +{ + public class TypeSpec + { + public TypeSpec(string name, int basicSize, IEnumerable slots, TypeFlags flags, int itemSize = 0) + { + this.Name = name ?? throw new ArgumentNullException(nameof(name)); + this.BasicSize = basicSize; + this.Slots = slots.ToArray(); + this.Flags = flags; + this.ItemSize = itemSize; + } + public string Name { get; } + public int BasicSize { get; } + public int ItemSize { get; } + public TypeFlags Flags { get; } + public IReadOnlyList Slots { get; } + + [StructLayout(LayoutKind.Sequential)] + public struct Slot + { + public Slot(TypeSlotID id, IntPtr value) + { + ID = id; + Value = value; + } + + public TypeSlotID ID { get; } + public IntPtr Value { get; } + } + } + + public enum TypeSlotID : int + { + mp_ass_subscript = 3, + mp_length = 4, + mp_subscript = 5, + nb_absolute = 6, + nb_add = 7, + nb_and = 8, + nb_bool = 9, + nb_divmod = 10, + nb_float = 11, + nb_floor_divide = 12, + nb_index = 13, + nb_inplace_add = 14, + nb_inplace_and = 15, + nb_inplace_floor_divide = 16, + nb_inplace_lshift = 17, + nb_inplace_multiply = 18, + nb_inplace_or = 19, + nb_inplace_power = 20, + nb_inplace_remainder = 21, + nb_inplace_rshift = 22, + nb_inplace_subtract = 23, + nb_inplace_true_divide = 24, + nb_inplace_xor = 25, + nb_int = 26, + nb_invert = 27, + nb_lshift = 28, + nb_multiply = 29, + nb_negative = 30, + nb_or = 31, + nb_positive = 32, + nb_power = 33, + nb_remainder = 34, + nb_rshift = 35, + nb_subtract = 36, + nb_true_divide = 37, + nb_xor = 38, + sq_ass_item = 39, + sq_concat = 40, + sq_contains = 41, + sq_inplace_concat = 42, + sq_inplace_repeat = 43, + sq_item = 44, + sq_length = 45, + sq_repeat = 46, + tp_alloc = 47, + tp_base = 48, + tp_bases = 49, + tp_call = 50, + tp_clear = 51, + tp_dealloc = 52, + tp_del = 53, + tp_descr_get = 54, + tp_descr_set = 55, + tp_doc = 56, + tp_getattr = 57, + tp_getattro = 58, + tp_hash = 59, + tp_init = 60, + tp_is_gc = 61, + tp_iter = 62, + tp_iternext = 63, + tp_methods = 64, + tp_new = 65, + tp_repr = 66, + tp_richcompare = 67, + tp_setattr = 68, + tp_setattro = 69, + tp_str = 70, + tp_traverse = 71, + tp_members = 72, + tp_getset = 73, + tp_free = 74, + nb_matrix_multiply = 75, + nb_inplace_matrix_multiply = 76, + am_await = 77, + am_aiter = 78, + am_anext = 79, + /// New in 3.5 + tp_finalize = 80, + } +} diff --git a/src/runtime/native/NativeTypeSpec.cs b/src/runtime/native/NativeTypeSpec.cs new file mode 100644 index 000000000..d55b77381 --- /dev/null +++ b/src/runtime/native/NativeTypeSpec.cs @@ -0,0 +1,45 @@ +#nullable enable +using System; +using System.Runtime.InteropServices; +using System.Text; + +namespace Python.Runtime.Native +{ + [StructLayout(LayoutKind.Sequential)] + struct NativeTypeSpec : IDisposable + { + public readonly StrPtr Name; + public readonly int BasicSize; + public readonly int ItemSize; + public readonly TypeFlags Flags; + public IntPtr Slots; + + public NativeTypeSpec(TypeSpec spec) + { + if (spec is null) throw new ArgumentNullException(nameof(spec)); + + this.Name = new StrPtr(spec.Name, Encoding.UTF8); + this.BasicSize = spec.BasicSize; + this.ItemSize = spec.ItemSize; + this.Flags = spec.Flags; + + unsafe + { + int slotsBytes = checked((spec.Slots.Count + 1) * Marshal.SizeOf()); + var slots = (TypeSpec.Slot*)Marshal.AllocHGlobal(slotsBytes); + for (int slotIndex = 0; slotIndex < spec.Slots.Count; slotIndex++) + slots[slotIndex] = spec.Slots[slotIndex]; + slots[spec.Slots.Count] = default; + this.Slots = (IntPtr)slots; + } + } + + public void Dispose() + { + // we have to leak the name + // this.Name.Dispose(); + Marshal.FreeHGlobal(this.Slots); + this.Slots = IntPtr.Zero; + } + } +} diff --git a/src/runtime/pytype.cs b/src/runtime/pytype.cs new file mode 100644 index 000000000..8bc08b76d --- /dev/null +++ b/src/runtime/pytype.cs @@ -0,0 +1,50 @@ +#nullable enable +using System; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + public class PyType : PyObject + { + /// Creates heap type object from the . + public PyType(TypeSpec spec, PyTuple? bases = null) : base(FromSpec(spec, bases)) { } + /// Wraps an existing type object. + public PyType(PyObject o) : base(FromObject(o)) { } + + /// Checks if specified object is a Python type. + public static bool IsType(PyObject value) + { + if (value is null) throw new ArgumentNullException(nameof(value)); + + return Runtime.PyType_Check(value.obj); + } + + private static BorrowedReference FromObject(PyObject o) + { + if (o is null) throw new ArgumentNullException(nameof(o)); + if (!IsType(o)) throw new ArgumentException("object is not a type"); + + return o.Reference; + } + + private static IntPtr FromSpec(TypeSpec spec, PyTuple? bases = null) + { + if (spec is null) throw new ArgumentNullException(nameof(spec)); + + if ((spec.Flags & TypeFlags.HeapType) == 0) + throw new NotSupportedException("Only heap types are supported"); + + var nativeSpec = new NativeTypeSpec(spec); + var basesRef = bases is null ? default : bases.Reference; + var result = Runtime.PyType_FromSpecWithBases(in nativeSpec, basesRef); + + PythonException.ThrowIfIsNull(result); + + nativeSpec.Dispose(); + + return result.DangerousMoveToPointer(); + } + } +} diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index b779c6707..caa160bcf 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -2006,6 +2006,8 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) private static IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n) => Delegates.PyType_GenericAlloc(type, n); + + internal static NewReference PyType_FromSpecWithBases(in NativeTypeSpec spec, BorrowedReference bases) => Delegates.PyType_FromSpecWithBases(in spec, bases); /// /// Finalize a type object. This should be called on all type objects to finish their initialization. This function is responsible for adding inherited slots from a type’s base class. Return 0 on success, or return -1 and sets an exception on error. @@ -2509,6 +2511,7 @@ static Delegates() PyException_SetCause = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyException_SetCause), GetUnmanagedDll(_PythonDll)); PyThreadState_SetAsyncExcLLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); PyThreadState_SetAsyncExcLP64 = (delegate* unmanaged[Cdecl])GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll)); + PyType_FromSpecWithBases = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyType_FromSpecWithBases), GetUnmanagedDll(PythonDLL)); } static global::System.IntPtr GetUnmanagedDll(string libraryName) @@ -2786,6 +2789,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLLP64 { get; } internal static delegate* unmanaged[Cdecl] PyThreadState_SetAsyncExcLP64 { get; } internal static delegate* unmanaged[Cdecl] PyObject_GenericGetDict { get; } + internal static delegate* unmanaged[Cdecl] PyType_FromSpecWithBases { get; } } }