Skip to content

Start to implement PyType_FromSpec type approach #1196

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 13 commits into from
Closed

Start to implement PyType_FromSpec type approach #1196

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
27fb3c3
Start to implement PyType_FromSpec type approach
koubaa Aug 1, 2020
bd26014
remove unused code
koubaa Aug 1, 2020
81ba32c
always add null slot
koubaa Aug 1, 2020
44d364e
change CreateType
koubaa Aug 1, 2020
1eb7f32
add comment
koubaa Aug 1, 2020
5a2ccff
some progress & slight refactoring
koubaa Aug 2, 2020
68b9d22
implement some PR suggestions
koubaa Aug 3, 2020
71c20f2
fix memory layout
koubaa Aug 4, 2020
5a8da7b
remove static constructor
koubaa Aug 4, 2020
3587638
Use struct
koubaa Aug 7, 2020
79d6a66
push missed change
koubaa Aug 7, 2020
0e74e0d
rename interop slots
koubaa Aug 7, 2020
6333888
re-add commented block
koubaa Aug 8, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions src/runtime/interop.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,11 @@ public static int TypeDictOffset(IntPtr type)
return ManagedDataOffsets.DictOffset(type);
}

public static int Size()
{
return size;
}

public static int Size(IntPtr pyType)
{
if (IsException(pyType))
Expand Down
4 changes: 2 additions & 2 deletions src/runtime/interop34.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,9 +131,9 @@ public static int magic()
public static int sq_inplace_repeat = 0;
public static int bf_getbuffer = 0;
public static int bf_releasebuffer = 0;
public static int name = 0;
public static int ht_name = 0;
public static int ht_slots = 0;
public static int qualname = 0;
public static int ht_qualname = 0;
public static int ht_cached_keys = 0;

/* here are optional user slots, followed by the members. */
Expand Down
4 changes: 2 additions & 2 deletions src/runtime/interop35.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,9 @@ public static int magic()
public static int sq_inplace_repeat = 0;
public static int bf_getbuffer = 0;
public static int bf_releasebuffer = 0;
public static int name = 0;
public static int ht_name = 0;
public static int ht_slots = 0;
public static int qualname = 0;
public static int ht_qualname = 0;
public static int ht_cached_keys = 0;

/* here are optional user slots, followed by the members. */
Expand Down
4 changes: 2 additions & 2 deletions src/runtime/interop36.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,9 @@ public static int magic()
public static int sq_inplace_repeat = 0;
public static int bf_getbuffer = 0;
public static int bf_releasebuffer = 0;
public static int name = 0;
public static int ht_name = 0;
public static int ht_slots = 0;
public static int qualname = 0;
public static int ht_qualname = 0;
public static int ht_cached_keys = 0;

/* here are optional user slots, followed by the members. */
Expand Down
4 changes: 2 additions & 2 deletions src/runtime/interop37.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,9 @@ public static int magic()
public static int sq_inplace_repeat = 0;
public static int bf_getbuffer = 0;
public static int bf_releasebuffer = 0;
public static int name = 0;
public static int ht_name = 0;
public static int ht_slots = 0;
public static int qualname = 0;
public static int ht_qualname = 0;
public static int ht_cached_keys = 0;

/* here are optional user slots, followed by the members. */
Expand Down
4 changes: 2 additions & 2 deletions src/runtime/interop38.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,9 +139,9 @@ public static int magic()
public static int sq_inplace_repeat = 0;
public static int bf_getbuffer = 0;
public static int bf_releasebuffer = 0;
public static int name = 0;
public static int ht_name = 0;
public static int ht_slots = 0;
public static int qualname = 0;
public static int ht_qualname = 0;
public static int ht_cached_keys = 0;

/* here are optional user slots, followed by the members. */
Expand Down
3 changes: 3 additions & 0 deletions src/runtime/runtime.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1726,6 +1726,9 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n)
return PyType_GenericAlloc(type, new IntPtr(n));
}

[DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)]
internal static extern NewReference PyType_FromSpec(ref TypeManager.PyTypeSpec spec);

[DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)]
private static extern IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n);

Expand Down
270 changes: 256 additions & 14 deletions src/runtime/typemanager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,23 +81,24 @@ internal static IntPtr GetTypeHandle(ManagedType obj, Type type)
/// </summary>
internal static IntPtr CreateType(Type impl)
{
IntPtr type = AllocateTypeObject(impl.Name);
int ob_size = ObjectOffset.Size(type);
var slotArray = CreateSlotArray(impl);
int flags = TypeFlags.Default | TypeFlags.Managed |
TypeFlags.HeapType | TypeFlags.HaveGC;

// Set tp_basicsize to the size of our managed instance objects.
Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size);
IntPtr type = CreateTypeObject(impl.Name, ObjectOffset.Size(), flags, slotArray);

if (ObjectOffset.Size() != ObjectOffset.Size(type))
{
//should we reset the size and call PyType_Ready again??
//how do we deal with the fact that size is based on whether
//the type is an exception type. Should CreateSlotArray
//return a tuple with both the slot array and a flag on
//whether the type array describes an exception or not?
}

var offset = (IntPtr)ObjectOffset.TypeDictOffset(type);
Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset);

InitializeSlots(type, impl);

int flags = TypeFlags.Default | TypeFlags.Managed |
TypeFlags.HeapType | TypeFlags.HaveGC;
Util.WriteCLong(type, TypeOffset.tp_flags, flags);

Runtime.PyType_Ready(type);

IntPtr dict = Marshal.ReadIntPtr(type, TypeOffset.tp_dict);
IntPtr mod = Runtime.PyString_FromString("CLR");
Runtime.PyDict_SetItemString(dict, "__module__", mod);
Expand Down Expand Up @@ -202,6 +203,17 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType)
return type;
}

static PY_TYPE_SLOT InitializeSlot(TypeSlots slotNumber, MethodInfo method)
{
var thunk = Interop.GetThunk(method);
return new PY_TYPE_SLOT { slot = slotNumber, func = thunk.Address};
}

static PY_TYPE_SLOT InitializeSlot(TypeSlots slotNumber, IntPtr thunk)
{
return new PY_TYPE_SLOT { slot = slotNumber, func = thunk };
}

static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method)
{
var thunk = Interop.GetThunk(method);
Expand Down Expand Up @@ -422,6 +434,163 @@ internal static IntPtr BasicSubType(string name, IntPtr base_, Type impl)
return type;
}

internal enum TypeSlots : int
{
bf_getbuffer = 1,
bf_releasebuffer = 2,
mp_ass_subscript = 3,
mp_length = 4,
mp_subscript = 5,
nb_absolute = 6,
nb_add = 7,
nb_and = 8,
nb_bool = 9,
nb_divmod = 10,
nb_float = 11,
nb_floor_divide = 12,
nb_index = 13,
nb_inplace_add = 14,
nb_inplace_and = 15,
nb_inplace_floor_divide = 16,
nb_inplace_lshift = 17,
nb_inplace_multiply = 18,
nb_inplace_or = 19,
nb_inplace_power = 20,
nb_inplace_remainder = 21,
nb_inplace_rshift = 22,
nb_inplace_subtract = 23,
nb_inplace_true_divide = 24,
nb_inplace_xor = 25,
nb_int = 26,
nb_invert = 27,
nb_lshift = 28,
nb_multiply = 29,
nb_negative = 30,
nb_or = 31,
nb_positive = 32,
nb_power = 33,
nb_remainder = 34,
nb_rshift = 35,
nb_subtract = 36,
nb_true_divide = 37,
nb_xor = 38,
sq_ass_item = 39,
sq_concat = 40,
sq_contains = 41,
sq_inplace_concat = 42,
sq_inplace_repeat = 43,
sq_item = 44,
sq_length = 45,
sq_repeat = 46,
tp_alloc = 47,
tp_base = 48,
tp_bases = 49,
tp_call = 50,
tp_clear = 51,
tp_dealloc = 52,
tp_del = 53,
tp_descr_get = 54,
tp_descr_set = 55,
tp_doc = 56,
tp_getattr = 57,
tp_getattro = 58,
tp_hash = 59,
tp_init = 60,
tp_is_gc = 61,
tp_iter = 62,
tp_iternext = 63,
tp_methods = 64,
tp_new = 65,
tp_repr = 66,
tp_richcompare = 67,
tp_setattr = 68,
tp_setattro = 69,
tp_str = 70,
tp_traverse = 71,
tp_members = 72,
tp_getset = 73,
tp_free = 74,
nb_matrix_multiply = 75,
nb_inplace_matrix_multiply = 76,
am_await = 77,
am_aiter = 78,
am_anext = 79,
tp_finalize = 80,
}

private static TypeSlots getSlotNumber(string methodName)
{
return (TypeSlots)Enum.Parse(typeof(TypeSlots), methodName);
}
Comment on lines +521 to +524
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't like to use string to get some constant information if it's not necessary, not to mention it use reflection for just getting the enum value.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@amos402 what do you propose?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just use the enum value directly.


[StructLayout(LayoutKind.Sequential)]
internal struct PY_TYPE_SLOT
{
internal TypeSlots slot; //slot id, from typeslots.h
internal IntPtr func; //function pointer of the function implementing the slot
}

[StructLayout(LayoutKind.Sequential)]
internal struct PyTypeSpec
{
public IntPtr Name;
public int BasicSize;
public int ItemSize;
public int Flags;
public IntPtr Slots;
}

internal static IntPtr CreateTypeObject(string name, int obSize, int obFlags, PY_TYPE_SLOT[] type_slots)
{
//convert type slot array to PyType_Slot*
int structSize = Marshal.SizeOf(typeof(PY_TYPE_SLOT));
GCHandle pinnedArray = GCHandle.Alloc(type_slots, GCHandleType.Pinned);
IntPtr slotsPtr = pinnedArray.AddrOfPinnedObject();

//convert name to char*
byte[] ascii = System.Text.Encoding.ASCII.GetBytes(name);
GCHandle pinnedName = GCHandle.Alloc(ascii, GCHandleType.Pinned);
IntPtr namePtr = pinnedName.AddrOfPinnedObject();

var typeSpec = new PyTypeSpec
{
Name = namePtr,
BasicSize = obSize,
ItemSize = 0,
Flags = obFlags,
Slots = slotsPtr
};

var type = Runtime.PyType_FromSpec(ref typeSpec).DangerousGetAddress();
pinnedArray.Free();
pinnedName.Free();

// Cheat a little: we'll set tp_name to the internal char * of
// the Python version of the type name - otherwise we'd have to
// allocate the tp_name and would have no way to free it.
IntPtr temp = Runtime.PyUnicode_FromString(name);
IntPtr raw = Runtime.PyUnicode_AsUTF8(temp);
Marshal.WriteIntPtr(type, TypeOffset.tp_name, raw);
Marshal.WriteIntPtr(type, TypeOffset.ht_name, temp);

Marshal.WriteIntPtr(type, TypeOffset.ht_qualname, temp);

long ptr = type.ToInt64(); // 64-bit safe

temp = new IntPtr(ptr + TypeOffset.nb_add);
Marshal.WriteIntPtr(type, TypeOffset.tp_as_number, temp);

temp = new IntPtr(ptr + TypeOffset.sq_length);
Marshal.WriteIntPtr(type, TypeOffset.tp_as_sequence, temp);

temp = new IntPtr(ptr + TypeOffset.mp_length);
Marshal.WriteIntPtr(type, TypeOffset.tp_as_mapping, temp);

temp = new IntPtr(ptr + TypeOffset.bf_getbuffer);
Marshal.WriteIntPtr(type, TypeOffset.tp_as_buffer, temp);

return type;
}

/// <summary>
/// Utility method to allocate a type object &amp; do basic initialization.
Expand All @@ -436,9 +605,9 @@ internal static IntPtr AllocateTypeObject(string name)
IntPtr temp = Runtime.PyUnicode_FromString(name);
IntPtr raw = Runtime.PyUnicode_AsUTF8(temp);
Marshal.WriteIntPtr(type, TypeOffset.tp_name, raw);
Marshal.WriteIntPtr(type, TypeOffset.name, temp);
Marshal.WriteIntPtr(type, TypeOffset.ht_name, temp);

Marshal.WriteIntPtr(type, TypeOffset.qualname, temp);
Marshal.WriteIntPtr(type, TypeOffset.ht_qualname, temp);

long ptr = type.ToInt64(); // 64-bit safe

Expand Down Expand Up @@ -680,6 +849,79 @@ internal static void InitializeNativeCodePage()
}
#endregion

/// <summary>
/// Given a managed Type that provides the implementation for the type,
/// create a PY_TYPE_SLOT array to be used for PyType_FromSpec.
/// </summary>
internal static PY_TYPE_SLOT[] CreateSlotArray(Type impl)
{
// We work from the most-derived class up; make sure to get
// the most-derived slot and not to override it with a base
// class's slot.
var seen = new HashSet<string>();
var typeslots = new List<PY_TYPE_SLOT>();

while (impl != null)
{
MethodInfo[] methods = impl.GetMethods(tbFlags);
foreach (MethodInfo method in methods)
{
string name = method.Name;
if (!(name.StartsWith("tp_") ||
name.StartsWith("nb_") ||
name.StartsWith("sq_") ||
name.StartsWith("mp_") ||
name.StartsWith("bf_")
))
{
continue;
}

if (seen.Contains(name))
{
continue;
}

typeslots.Add(InitializeSlot(getSlotNumber(name), method));
seen.Add(name);
}

impl = impl.BaseType;
}

var native = NativeCode.Active;

// The garbage collection related slots always have to return 1 or 0
// since .NET objects don't take part in Python's gc:
// tp_traverse (returns 0)
// tp_clear (returns 0)
// tp_is_gc (returns 1)
// These have to be defined, though, so by default we fill these with
// static C# functions from this class.

var ret0 = Interop.GetThunk(((Func<IntPtr, int>)Return0).Method).Address;
var ret1 = Interop.GetThunk(((Func<IntPtr, int>)Return1).Method).Address;

if (native != null)
{
// If we want to support domain reload, the C# implementation
// cannot be used as the assembly may get released before
// CPython calls these functions. Instead, for amd64 and x86 we
// load them into a separate code page that is leaked
// intentionally.
InitializeNativeCodePage();
ret1 = NativeCodePage + native.Return1;
ret0 = NativeCodePage + native.Return0;
}

typeslots.Add(InitializeSlot(getSlotNumber("tp_traverse"), ret0));
typeslots.Add(InitializeSlot(getSlotNumber("tp_clear"), ret0));
typeslots.Add(InitializeSlot(getSlotNumber("tp_is_gc"), ret1));

typeslots.Add(new PY_TYPE_SLOT { slot = 0, func = IntPtr.Zero });
return typeslots.ToArray();
}

/// <summary>
/// Given a newly allocated Python type object and a managed Type that
/// provides the implementation for the type, connect the type slots of
Expand Down