Skip to content

Detect Py_TRACE_REFS at runtime and calculate object offsets accordingly #1426

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 27, 2021
Merged

Detect Py_TRACE_REFS at runtime and calculate object offsets accordingly #1426

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions src/runtime/classderived.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,10 +102,7 @@ internal static IntPtr ToPython(IPythonDerivedType obj)
// collected while Python still has a reference to it.
if (Runtime.Refcount(self.pyHandle) == 1)
{

#if PYTHON_WITH_PYDEBUG
Runtime._Py_NewReference(self.pyHandle);
#endif
Runtime._Py_NewReference(self.ObjectReference);
GCHandle gc = GCHandle.Alloc(self, GCHandleType.Normal);
SetGCHandle(self.ObjectReference, self.TypeReference, gc);
self.gcHandle.Free();
Expand Down
10 changes: 10 additions & 0 deletions src/runtime/converter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -525,6 +525,16 @@ internal static bool ToManagedValue(IntPtr value, Type obType,

internal delegate bool TryConvertFromPythonDelegate(IntPtr pyObj, out object result);

internal static int ToInt32(BorrowedReference value)
{
nint num = Runtime.PyLong_AsSignedSize_t(value);
if (num == -1 && Exceptions.ErrorOccurred())
{
throw new PythonException();
}
return checked((int)num);
}

/// <summary>
/// Convert a Python value to an instance of a primitive managed type.
/// </summary>
Expand Down
25 changes: 22 additions & 3 deletions src/runtime/native/ABI.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ namespace Python.Runtime.Native

static class ABI
{
public static int RefCountOffset { get; } = GetRefCountOffset();
public static int ObjectHeadOffset => RefCountOffset;

internal static void Initialize(Version version, BorrowedReference pyType)
{
string offsetsClassSuffix = string.Format(CultureInfo.InvariantCulture,
Expand All @@ -16,18 +19,34 @@ internal static void Initialize(Version version, BorrowedReference pyType)

const string nativeTypeOffsetClassName = "Python.Runtime.NativeTypeOffset";
string className = "Python.Runtime.TypeOffset" + offsetsClassSuffix;
Type nativeOffsetsClass = thisAssembly.GetType(nativeTypeOffsetClassName, throwOnError: false);
Type typeOffsetsClass =
// Try platform native offsets first. It is only present when generated by setup.py
thisAssembly.GetType(nativeTypeOffsetClassName, throwOnError: false)
?? thisAssembly.GetType(className, throwOnError: false);
nativeOffsetsClass ?? thisAssembly.GetType(className, throwOnError: false);
if (typeOffsetsClass is null)
{
var types = thisAssembly.GetTypes().Select(type => type.Name).Where(name => name.StartsWith("TypeOffset"));
string message = $"Searching for {className}, found {string.Join(",", types)}.";
throw new NotSupportedException($"Python ABI v{version} is not supported: {message}");
}

var typeOffsets = (ITypeOffsets)Activator.CreateInstance(typeOffsetsClass);
TypeOffset.Use(typeOffsets);
TypeOffset.Use(typeOffsets, nativeOffsetsClass == null ? ObjectHeadOffset : 0);
}

static unsafe int GetRefCountOffset()
{
IntPtr tempObject = Runtime.PyList_New(0);
IntPtr* tempPtr = (IntPtr*)tempObject;
int offset = 0;
while(tempPtr[offset] != (IntPtr)1)
{
offset++;
if (offset > 100)
throw new InvalidProgramException("PyObject_HEAD could not be found withing reasonable distance from the start of PyObject");
}
Runtime.XDecref(tempObject);
return offset * IntPtr.Size;
}
}
}
3 changes: 2 additions & 1 deletion src/runtime/native/TypeOffset.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ static partial class TypeOffset
internal static int tp_str { get; private set; }
internal static int tp_traverse { get; private set; }

internal static void Use(ITypeOffsets offsets)
internal static void Use(ITypeOffsets offsets, int extraHeadOffset)
{
if (offsets is null) throw new ArgumentNullException(nameof(offsets));

Expand All @@ -87,6 +87,7 @@ internal static void Use(ITypeOffsets offsets)

var sourceProperty = typeof(ITypeOffsets).GetProperty(offsetProperty.Name);
int value = (int)sourceProperty.GetValue(offsets, null);
value += extraHeadOffset;
offsetProperty.SetValue(obj: null, value: value, index: null);
}

Expand Down
99 changes: 56 additions & 43 deletions src/runtime/runtime.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,9 +85,9 @@ internal static Version PyVersion
{
using (var versionTuple = new PyTuple(PySys_GetObject("version_info")))
{
var major = versionTuple[0].As<int>();
var minor = versionTuple[1].As<int>();
var micro = versionTuple[2].As<int>();
var major = Converter.ToInt32(versionTuple[0].Reference);
var minor = Converter.ToInt32(versionTuple[1].Reference);
var micro = Converter.ToInt32(versionTuple[2].Reference);
return new Version(major, minor, micro);
}
}
Expand Down Expand Up @@ -198,15 +198,15 @@ private static void InitPyMembers()
SetPyMember(ref PyFalse, PyObject_GetAttrString(builtins, "False"),
() => PyFalse = IntPtr.Zero);

SetPyMember(ref PyBoolType, PyObject_Type(PyTrue),
SetPyMemberTypeOf(ref PyBoolType, PyTrue,
() => PyBoolType = IntPtr.Zero);
SetPyMember(ref PyNoneType, PyObject_Type(PyNone),
SetPyMemberTypeOf(ref PyNoneType, PyNone,
() => PyNoneType = IntPtr.Zero);
SetPyMember(ref PyTypeType, PyObject_Type(PyNoneType),
SetPyMemberTypeOf(ref PyTypeType, PyNoneType,
() => PyTypeType = IntPtr.Zero);

op = PyObject_GetAttrString(builtins, "len");
SetPyMember(ref PyMethodType, PyObject_Type(op),
SetPyMemberTypeOf(ref PyMethodType, op,
() => PyMethodType = IntPtr.Zero);
XDecref(op);

Expand All @@ -215,7 +215,7 @@ private static void InitPyMembers()
//
// object.__init__ seems safe, though.
op = PyObject_GetAttr(PyBaseObjectType, PyIdentifier.__init__);
SetPyMember(ref PyWrapperDescriptorType, PyObject_Type(op),
SetPyMemberTypeOf(ref PyWrapperDescriptorType, op,
() => PyWrapperDescriptorType = IntPtr.Zero);
XDecref(op);

Expand All @@ -226,47 +226,47 @@ private static void InitPyMembers()
}

op = PyString_FromString("string");
SetPyMember(ref PyStringType, PyObject_Type(op),
SetPyMemberTypeOf(ref PyStringType, op,
() => PyStringType = IntPtr.Zero);
XDecref(op);

op = PyUnicode_FromString("unicode");
SetPyMember(ref PyUnicodeType, PyObject_Type(op),
SetPyMemberTypeOf(ref PyUnicodeType, op,
() => PyUnicodeType = IntPtr.Zero);
XDecref(op);

op = EmptyPyBytes();
SetPyMember(ref PyBytesType, PyObject_Type(op),
SetPyMemberTypeOf(ref PyBytesType, op,
() => PyBytesType = IntPtr.Zero);
XDecref(op);

op = PyTuple_New(0);
SetPyMember(ref PyTupleType, PyObject_Type(op),
SetPyMemberTypeOf(ref PyTupleType, op,
() => PyTupleType = IntPtr.Zero);
XDecref(op);

op = PyList_New(0);
SetPyMember(ref PyListType, PyObject_Type(op),
SetPyMemberTypeOf(ref PyListType, op,
() => PyListType = IntPtr.Zero);
XDecref(op);

op = PyDict_New();
SetPyMember(ref PyDictType, PyObject_Type(op),
SetPyMemberTypeOf(ref PyDictType, op,
() => PyDictType = IntPtr.Zero);
XDecref(op);

op = PyInt_FromInt32(0);
SetPyMember(ref PyIntType, PyObject_Type(op),
SetPyMemberTypeOf(ref PyIntType, op,
() => PyIntType = IntPtr.Zero);
XDecref(op);

op = PyLong_FromLong(0);
SetPyMember(ref PyLongType, PyObject_Type(op),
SetPyMemberTypeOf(ref PyLongType, op,
() => PyLongType = IntPtr.Zero);
XDecref(op);

op = PyFloat_FromDouble(0);
SetPyMember(ref PyFloatType, PyObject_Type(op),
SetPyMemberTypeOf(ref PyFloatType, op,
() => PyFloatType = IntPtr.Zero);
XDecref(op);

Expand All @@ -278,7 +278,8 @@ private static void InitPyMembers()
_PyObject_NextNotImplemented = Get_PyObject_NextNotImplemented();
{
using var sys = PyImport_ImportModule("sys");
PyModuleType = PyObject_Type(sys.DangerousMoveToPointer());
SetPyMemberTypeOf(ref PyModuleType, sys.DangerousGetAddress(),
() => PyModuleType = IntPtr.Zero);
}
}

Expand Down Expand Up @@ -455,6 +456,12 @@ private static void SetPyMember(ref IntPtr obj, IntPtr value, Action onRelease)
_pyRefs.Add(value, onRelease);
}

private static void SetPyMemberTypeOf(ref IntPtr obj, IntPtr value, Action onRelease)
{
var type = PyObject_Type(new BorrowedReference(value)).DangerousMoveToPointer();
SetPyMember(ref obj, type, onRelease);
}

private static void ResetPyMembers()
{
_pyRefs.Release();
Expand Down Expand Up @@ -761,16 +768,12 @@ internal static unsafe void XDecref(IntPtr op)
[Pure]
internal static unsafe long Refcount(IntPtr op)
{
#if PYTHON_WITH_PYDEBUG
var p = (void*)(op + TypeOffset.ob_refcnt);
#else
var p = (void*)op;
#endif
if ((void*)0 == p)
if (op == IntPtr.Zero)
{
return 0;
}
return Is32Bit ? (*(int*)p) : (*(long*)p);
var p = (nint*)(op + ABI.RefCountOffset);
return *p;
}

/// <summary>
Expand Down Expand Up @@ -977,14 +980,9 @@ internal static unsafe IntPtr PyObject_TYPE(IntPtr op)
{
return IntPtr.Zero;
}
#if PYTHON_WITH_PYDEBUG
var n = 3;
#else
var n = 1;
#endif
return Is32Bit
? new IntPtr((void*)(*((uint*)p + n)))
: new IntPtr((void*)(*((ulong*)p + n)));
Debug.Assert(TypeOffset.ob_type > 0);
IntPtr* typePtr = (IntPtr*)(op + TypeOffset.ob_type);
return *typePtr;
}
internal static unsafe BorrowedReference PyObject_TYPE(BorrowedReference op)
=> new BorrowedReference(PyObject_TYPE(op.DangerousGetAddress()));
Expand All @@ -1001,6 +999,9 @@ internal static IntPtr PyObject_Type(IntPtr op)
return tp;
}

internal static NewReference PyObject_Type(BorrowedReference o)
=> Delegates.PyObject_Type(o);

internal static string PyObject_GetTypeName(IntPtr op)
{
IntPtr pyType = PyObject_TYPE(op);
Expand Down Expand Up @@ -1145,10 +1146,11 @@ internal static IntPtr PyObject_Str(IntPtr pointer)

internal static IntPtr PyObject_Dir(IntPtr pointer) => Delegates.PyObject_Dir(pointer);

#if PYTHON_WITH_PYDEBUG
[DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)]
internal static extern void _Py_NewReference(IntPtr ob);
#endif
internal static void _Py_NewReference(BorrowedReference ob)
{
if (Delegates._Py_NewReference != null)
Delegates._Py_NewReference(ob);
}

//====================================================================
// Python buffer API
Expand Down Expand Up @@ -1912,11 +1914,6 @@ internal static string PyModule_GetName(IntPtr module)
internal static string PyModule_GetFilename(IntPtr module)
=> Delegates.PyModule_GetFilename(module).ToString(Encoding.UTF8);

#if PYTHON_WITH_PYDEBUG
[DllImport(_PythonDll, EntryPoint = "PyModule_Create2TraceRefs", CallingConvention = CallingConvention.Cdecl)]
#else

#endif
internal static IntPtr PyModule_Create2(IntPtr module, int apiver) => Delegates.PyModule_Create2(module, apiver);


Expand Down Expand Up @@ -2331,6 +2328,7 @@ static Delegates()
PyObject_Hash = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyObject_Hash), GetUnmanagedDll(_PythonDll));
PyObject_Repr = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyObject_Repr), GetUnmanagedDll(_PythonDll));
PyObject_Str = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyObject_Str), GetUnmanagedDll(_PythonDll));
PyObject_Type = (delegate* unmanaged[Cdecl]<BorrowedReference, NewReference>)GetFunctionByName(nameof(PyObject_Type), GetUnmanagedDll(_PythonDll));
PyObject_Dir = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyObject_Dir), GetUnmanagedDll(_PythonDll));
PyObject_GetBuffer = (delegate* unmanaged[Cdecl]<IntPtr, ref Py_buffer, int, int>)GetFunctionByName(nameof(PyObject_GetBuffer), GetUnmanagedDll(_PythonDll));
PyBuffer_Release = (delegate* unmanaged[Cdecl]<ref Py_buffer, void>)GetFunctionByName(nameof(PyBuffer_Release), GetUnmanagedDll(_PythonDll));
Expand Down Expand Up @@ -2466,7 +2464,14 @@ static Delegates()
PyModule_GetName = (delegate* unmanaged[Cdecl]<IntPtr, StrPtr>)GetFunctionByName(nameof(PyModule_GetName), GetUnmanagedDll(_PythonDll));
PyModule_GetDict = (delegate* unmanaged[Cdecl]<BorrowedReference, BorrowedReference>)GetFunctionByName(nameof(PyModule_GetDict), GetUnmanagedDll(_PythonDll));
PyModule_GetFilename = (delegate* unmanaged[Cdecl]<IntPtr, StrPtr>)GetFunctionByName(nameof(PyModule_GetFilename), GetUnmanagedDll(_PythonDll));
PyModule_Create2 = (delegate* unmanaged[Cdecl]<IntPtr, int, IntPtr>)GetFunctionByName(nameof(PyModule_Create2), GetUnmanagedDll(_PythonDll));
try
{
PyModule_Create2 = (delegate* unmanaged[Cdecl]<IntPtr, int, IntPtr>)GetFunctionByName(nameof(PyModule_Create2), GetUnmanagedDll(_PythonDll));
}
catch (MissingMethodException)
{
PyModule_Create2 = (delegate* unmanaged[Cdecl]<IntPtr, int, IntPtr>)GetFunctionByName("PyModule_Create2TraceRefs", GetUnmanagedDll(_PythonDll));
}
PyImport_Import = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyImport_Import), GetUnmanagedDll(_PythonDll));
PyImport_ImportModule = (delegate* unmanaged[Cdecl]<StrPtr, NewReference>)GetFunctionByName(nameof(PyImport_ImportModule), GetUnmanagedDll(_PythonDll));
PyImport_ReloadModule = (delegate* unmanaged[Cdecl]<BorrowedReference, NewReference>)GetFunctionByName(nameof(PyImport_ReloadModule), GetUnmanagedDll(_PythonDll));
Expand Down Expand Up @@ -2521,6 +2526,12 @@ static Delegates()
PyThreadState_SetAsyncExcLLP64 = (delegate* unmanaged[Cdecl]<uint, IntPtr, int>)GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll));
PyThreadState_SetAsyncExcLP64 = (delegate* unmanaged[Cdecl]<ulong, IntPtr, int>)GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll));
PyType_FromSpecWithBases = (delegate* unmanaged[Cdecl]<in NativeTypeSpec, BorrowedReference, NewReference>)GetFunctionByName(nameof(PyType_FromSpecWithBases), GetUnmanagedDll(PythonDLL));

try
{
_Py_NewReference = (delegate* unmanaged[Cdecl]<BorrowedReference, void>)GetFunctionByName(nameof(_Py_NewReference), GetUnmanagedDll(_PythonDll));
}
catch (MissingMethodException) { }
}

static global::System.IntPtr GetUnmanagedDll(string libraryName)
Expand Down Expand Up @@ -2616,6 +2627,7 @@ static Delegates()
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> PyObject_Hash { get; }
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> PyObject_Repr { get; }
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> PyObject_Str { get; }
internal static delegate* unmanaged[Cdecl]<BorrowedReference, NewReference> PyObject_Type { get; }
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> PyObject_Dir { get; }
internal static delegate* unmanaged[Cdecl]<IntPtr, ref Py_buffer, int, int> PyObject_GetBuffer { get; }
internal static delegate* unmanaged[Cdecl]<ref Py_buffer, void> PyBuffer_Release { get; }
Expand Down Expand Up @@ -2799,6 +2811,7 @@ static Delegates()
internal static delegate* unmanaged[Cdecl]<ulong, IntPtr, int> PyThreadState_SetAsyncExcLP64 { get; }
internal static delegate* unmanaged[Cdecl]<BorrowedReference, IntPtr, NewReference> PyObject_GenericGetDict { get; }
internal static delegate* unmanaged[Cdecl]<in NativeTypeSpec, BorrowedReference, NewReference> PyType_FromSpecWithBases { get; }
internal static delegate* unmanaged[Cdecl]<BorrowedReference, void> _Py_NewReference { get; }
}
}

Expand Down