WIP: use C# 9 function pointers for PInvoke

pythonnet · lostmsu · Jan 28, 2021 · Nov 30, 2020 · Dec 2, 2020 · Dec 2, 2020
commit 972c41d018c78f08d2531c8e3ae640f3038e79d0
diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs
@@ -10,10 +10,11 @@ readonly ref struct BorrowedReference
         readonly IntPtr pointer;
         public bool IsNull => this.pointer == IntPtr.Zero;
 
-
         /// <summary>Gets a raw pointer to the Python object</summary>
         public IntPtr DangerousGetAddress()
             => this.IsNull ? throw new NullReferenceException() : this.pointer;
+        /// <summary>Gets a raw pointer to the Python object</summary>
+        public IntPtr DangerousGetAddressOrNull() => this.pointer;
 
         /// <summary>
         /// Creates new instance of <see cref="BorrowedReference"/> from raw pointer. Unsafe.

diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs
@@ -28,6 +28,16 @@ public PyObject MoveToPyObject()
             return result;
         }
 
+        /// <summary>Moves ownership of this instance to unmanged pointer</summary>
+        public IntPtr DangerousMoveToPointer()
+        {
+            if (this.IsNull()) throw new NullReferenceException();
+
+            var result = this.pointer;
+            this.pointer = IntPtr.Zero;
+            return result;
+        }
+
         /// <summary>Moves ownership of this instance to unmanged pointer</summary>
         public IntPtr DangerousMoveToPointerOrNull()
         {

diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj
@@ -2,6 +2,7 @@
   <PropertyGroup>
     <TargetFrameworks>netstandard2.0</TargetFrameworks>
     <Platforms>AnyCPU</Platforms>
+    <LangVersion>9.0</LangVersion>
     <RootNamespace>Python.Runtime</RootNamespace>
     <AssemblyName>Python.Runtime</AssemblyName>
     <LangVersion>9.0</LangVersion>

diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs
@@ -117,7 +117,7 @@ internal static void SaveRuntimeData(RuntimeDataStorage storage)
                 // Python object's dictionary tool; thus raising an AttributeError
                 // instead of a TypeError.
                 // Classes are re-initialized on in RestoreRuntimeData.
-                IntPtr dict = Marshal.ReadIntPtr(cls.Value.tpHandle, TypeOffset.tp_dict);
+                var dict = new BorrowedReference(Marshal.ReadIntPtr(cls.Value.tpHandle, TypeOffset.tp_dict));
                 foreach (var member in cls.Value.dotNetMembers)
                 {
                     // No need to decref the member, the ClassBase instance does 
@@ -269,7 +269,7 @@ private static void InitClassBase(Type type, ClassBase impl)
             IntPtr tp = TypeManager.GetTypeHandle(impl, type);
 
             // Finally, initialize the class __dict__ and return the object.
-            IntPtr dict = Marshal.ReadIntPtr(tp, TypeOffset.tp_dict);
+            var dict = new BorrowedReference(Marshal.ReadIntPtr(tp, TypeOffset.tp_dict));
 
 
             if (impl.dotNetMembers == null)
@@ -282,7 +282,7 @@ private static void InitClassBase(Type type, ClassBase impl)
                 var item = (ManagedType)iter.Value;
                 var name = (string)iter.Key;
                 impl.dotNetMembers.Add(name);
-                Runtime.PyDict_SetItemString(dict, name, item.pyHandle);
+                Runtime.PyDict_SetItemString(dict, name, item.ObjectReference);
                 // Decref the item now that it's been used.
                 item.DecrRefCount();
                 if (ClassBase.CilToPyOpMap.TryGetValue(name, out var pyOp)) {
@@ -291,20 +291,15 @@ private static void InitClassBase(Type type, ClassBase impl)
             }
 
             // If class has constructors, generate an __doc__ attribute.
-            IntPtr doc = IntPtr.Zero;
+            NewReference doc = default;
             Type marker = typeof(DocStringAttribute);
             var attrs = (Attribute[])type.GetCustomAttributes(marker, false);
-            if (attrs.Length == 0)
-            {
-                doc = IntPtr.Zero;
-            }
-            else
+            if (attrs.Length != 0)
             {
                 var attr = (DocStringAttribute)attrs[0];
                 string docStr = attr.DocString;
-                doc = Runtime.PyString_FromString(docStr);
+                doc = NewReference.DangerousFromPointer(Runtime.PyString_FromString(docStr));
                 Runtime.PyDict_SetItem(dict, PyIdentifier.__doc__, doc);
-                Runtime.XDecref(doc);
             }
 
             var co = impl as ClassObject;
@@ -320,20 +315,21 @@ private static void InitClassBase(Type type, ClassBase impl)
                         var ctors = new ConstructorBinding(type, tp, co.binder);
                         // ExtensionType types are untracked, so don't Incref() them.
                         // TODO: deprecate __overloads__ soon...
-                        Runtime.PyDict_SetItem(dict, PyIdentifier.__overloads__, ctors.pyHandle);
-                        Runtime.PyDict_SetItem(dict, PyIdentifier.Overloads, ctors.pyHandle);
+                        Runtime.PyDict_SetItem(dict, PyIdentifier.__overloads__, ctors.ObjectReference);
+                        Runtime.PyDict_SetItem(dict, PyIdentifier.Overloads, ctors.ObjectReference);
                         ctors.DecrRefCount();
                     }
 
                     // don't generate the docstring if one was already set from a DocStringAttribute.
-                    if (!CLRModule._SuppressDocs && doc == IntPtr.Zero)
+                    if (!CLRModule._SuppressDocs && doc.IsNull())
                     {
                         doc = co.GetDocString();
                         Runtime.PyDict_SetItem(dict, PyIdentifier.__doc__, doc);
-                        Runtime.XDecref(doc);
                     }
                 }
             }
+            doc.Dispose();
+
             // The type has been modified after PyType_Ready has been called
             // Refresh the type
             Runtime.PyType_Modified(tp);

diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs
@@ -31,7 +31,7 @@ internal ClassObject(Type tp) : base(tp)
         /// <summary>
         /// Helper to get docstring from reflected constructor info.
         /// </summary>
-        internal IntPtr GetDocString()
+        internal NewReference GetDocString()
         {
             MethodBase[] methods = binder.GetMethods();
             var str = "";
@@ -43,7 +43,7 @@ internal IntPtr GetDocString()
                 }
                 str += t.ToString();
             }
-            return Runtime.PyString_FromString(str);
+            return NewReference.DangerousFromPointer(Runtime.PyString_FromString(str));
         }
 
 

diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs
@@ -13,6 +13,7 @@ internal static class ImportHook
         private static CLRModule root;
         private static MethodWrapper hook;
         private static IntPtr py_clr_module;
+        static BorrowedReference ClrModuleReference => new BorrowedReference(py_clr_module);
 
         private static IntPtr module_def = IntPtr.Zero;
 
@@ -74,7 +75,7 @@ static void RestoreImport()
         /// <summary>
         /// Initialization performed on startup of the Python runtime.
         /// </summary>
-        internal static void Initialize()
+        internal static unsafe void Initialize()
         {
             InitImport();
 
@@ -86,14 +87,13 @@ internal static void Initialize()
             py_clr_module = Runtime.PyModule_Create2(module_def, 3);
 
             // both dicts are borrowed references
-            IntPtr mod_dict = Runtime.PyModule_GetDict(py_clr_module);
-            IntPtr clr_dict = Runtime._PyObject_GetDictPtr(root.pyHandle); // PyObject**
-            clr_dict = (IntPtr)Marshal.PtrToStructure(clr_dict, typeof(IntPtr));
+            BorrowedReference mod_dict = Runtime.PyModule_GetDict(ClrModuleReference);
+            BorrowedReference clr_dict = *Runtime._PyObject_GetDictPtr(root.ObjectReference);
 
             Runtime.PyDict_Update(mod_dict, clr_dict);
-            IntPtr dict = Runtime.PyImport_GetModuleDict();
-            Runtime.PyDict_SetItemString(dict, "CLR", py_clr_module);
-            Runtime.PyDict_SetItemString(dict, "clr", py_clr_module);
+            BorrowedReference dict = Runtime.PyImport_GetModuleDict();
+            Runtime.PyDict_SetItemString(dict, "CLR", ClrModuleReference);
+            Runtime.PyDict_SetItemString(dict, "clr", ClrModuleReference);
         }
 
 
@@ -143,67 +143,62 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage)
         /// <summary>
         /// Return the clr python module (new reference)
         /// </summary>
-        public static IntPtr GetCLRModule(IntPtr? fromList = null)
+        public static unsafe NewReference GetCLRModule(BorrowedReference fromList = default)
         {
             root.InitializePreload();
 
             // update the module dictionary with the contents of the root dictionary
             root.LoadNames();
-            IntPtr py_mod_dict = Runtime.PyModule_GetDict(py_clr_module);
-            IntPtr clr_dict = Runtime._PyObject_GetDictPtr(root.pyHandle); // PyObject**
-            clr_dict = (IntPtr)Marshal.PtrToStructure(clr_dict, typeof(IntPtr));
+            BorrowedReference py_mod_dict = Runtime.PyModule_GetDict(ClrModuleReference);
+            BorrowedReference clr_dict = *Runtime._PyObject_GetDictPtr(root.ObjectReference);
             Runtime.PyDict_Update(py_mod_dict, clr_dict);
 
             // find any items from the from list and get them from the root if they're not
             // already in the module dictionary
-            if (fromList != null && fromList != IntPtr.Zero)
+            if (fromList != null && fromList != default)
             {
-                if (Runtime.PyTuple_Check(fromList.GetValueOrDefault()))
+                if (Runtime.PyTuple_Check(fromList))
                 {
-                    Runtime.XIncref(py_mod_dict);
-                    using (var mod_dict = new PyDict(py_mod_dict))
+                    using var mod_dict = new PyDict(py_mod_dict);
+                    using var from = new PyTuple(fromList);
+                    foreach (PyObject item in from)
                     {
-                        Runtime.XIncref(fromList.GetValueOrDefault());
-                        using (var from = new PyTuple(fromList.GetValueOrDefault()))
+                        if (mod_dict.HasKey(item))
                         {
-                            foreach (PyObject item in from)
-                            {
-                                if (mod_dict.HasKey(item))
-                                {
-                                    continue;
-                                }
-
-                                var s = item.AsManagedObject(typeof(string)) as string;
-                                if (s == null)
-                                {
-                                    continue;
-                                }
-
-                                ManagedType attr = root.GetAttribute(s, true);
-                                if (attr == null)
-                                {
-                                    continue;
-                                }
-
-                                Runtime.XIncref(attr.pyHandle);
-                                using (var obj = new PyObject(attr.pyHandle))
-                                {
-                                    mod_dict.SetItem(s, obj);
-                                }
-                            }
+                            continue;
+                        }
+
+                        var s = item.AsManagedObject(typeof(string)) as string;
+                        if (s == null)
+                        {
+                            continue;
+                        }
+
+                        ManagedType attr = root.GetAttribute(s, true);
+                        if (attr == null)
+                        {
+                            continue;
+                        }
+
+                        Runtime.XIncref(attr.pyHandle);
+                        using (var obj = new PyObject(attr.pyHandle))
+                        {
+                            mod_dict.SetItem(s, obj);
                         }
                     }
                 }
             }
             Runtime.XIncref(py_clr_module);
-            return py_clr_module;
+            return NewReference.DangerousFromPointer(py_clr_module);
         }
 
         /// <summary>
         /// The actual import hook that ties Python to the managed world.
         /// </summary>
-        public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw)
+        public static IntPtr __import__(IntPtr self, IntPtr argsRaw, IntPtr kw)
         {
+            var args = new BorrowedReference(argsRaw);
+
             // Replacement for the builtin __import__. The original import
             // hook is saved as this.py_import. This version handles CLR
             // import and defers to the normal builtin for everything else.
@@ -214,9 +209,8 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw)
                 return Exceptions.RaiseTypeError("__import__() takes at least 1 argument (0 given)");
             }
 
-            // borrowed reference
-            IntPtr py_mod_name = Runtime.PyTuple_GetItem(args, 0);
-            if (py_mod_name == IntPtr.Zero ||
+            BorrowedReference py_mod_name = Runtime.PyTuple_GetItem(args, 0);
+            if (py_mod_name.IsNull ||
                 !Runtime.IsStringType(py_mod_name))
             {
                 return Exceptions.RaiseTypeError("string expected");
@@ -225,12 +219,12 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw)
             // Check whether the import is of the form 'from x import y'.
             // This determines whether we return the head or tail module.
 
-            IntPtr fromList = IntPtr.Zero;
+            BorrowedReference fromList = default;
             var fromlist = false;
             if (num_args >= 4)
             {
                 fromList = Runtime.PyTuple_GetItem(args, 3);
-                if (fromList != IntPtr.Zero &&
+                if (fromList != default &&
                     Runtime.PyObject_IsTrue(fromList) == 1)
                 {
                     fromlist = true;
@@ -243,16 +237,16 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw)
             // the module.
             if (mod_name == "clr")
             {
-                IntPtr clr_module = GetCLRModule(fromList);
-                if (clr_module != IntPtr.Zero)
+                NewReference clr_module = GetCLRModule(fromList);
+                if (!clr_module.IsNull())
                 {
-                    IntPtr sys_modules = Runtime.PyImport_GetModuleDict();
-                    if (sys_modules != IntPtr.Zero)
+                    BorrowedReference sys_modules = Runtime.PyImport_GetModuleDict();
+                    if (!sys_modules.IsNull)
                     {
                         Runtime.PyDict_SetItemString(sys_modules, "clr", clr_module);
                     }
                 }
-                return clr_module;
+                return clr_module.DangerousMoveToPointerOrNull();
             }
 
             string realname = mod_name;
@@ -265,7 +259,7 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw)
             // Turns out that the AssemblyManager.ResolveHandler() checks to see if any
             // Assembly's FullName.ToLower().StartsWith(name.ToLower()), which makes very
             // little sense to me.
-            IntPtr res = Runtime.PyObject_Call(py_import, args, kw);
+            IntPtr res = Runtime.PyObject_Call(py_import, args.DangerousGetAddress(), kw);
             if (res != IntPtr.Zero)
             {
                 // There was no error.
@@ -300,10 +294,10 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw)
 
             // See if sys.modules for this interpreter already has the
             // requested module. If so, just return the existing module.
-            IntPtr modules = Runtime.PyImport_GetModuleDict();
-            IntPtr module = Runtime.PyDict_GetItem(modules, py_mod_name);
+            BorrowedReference modules = Runtime.PyImport_GetModuleDict();
+            BorrowedReference module = Runtime.PyDict_GetItem(modules, py_mod_name);
 
-            if (module != IntPtr.Zero)
+            if (module != default)
             {
                 if (fromlist)
                 {
@@ -312,16 +306,15 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw)
                         var mod = ManagedType.GetManagedObject(module) as ModuleObject;
                         mod?.LoadNames();
                     }
-                    Runtime.XIncref(module);
-                    return module;
+                    return Runtime.NewRef(module).DangerousMoveToPointer();
                 }
                 if (clr_prefix != null)
                 {
-                    return GetCLRModule(fromList);
+                    return GetCLRModule(fromList).DangerousMoveToPointerOrNull();
                 }
                 module = Runtime.PyDict_GetItemString(modules, names[0]);
-                Runtime.XIncref(module);
-                return module;
+                return Runtime.NewRefOrNull(module)
+                              .DangerousMoveToPointer();
             }
             Exceptions.Clear();
 
@@ -358,12 +351,12 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw)
                 }
 
                 // Add the module to sys.modules
-                Runtime.PyDict_SetItemString(modules, tail.moduleName, tail.pyHandle);
+                Runtime.PyDict_SetItemString(modules, tail.moduleName, tail.ObjectReference);
 
                 // If imported from CLR add clr.<modulename> to sys.modules as well
                 if (clr_prefix != null)
                 {
-                    Runtime.PyDict_SetItemString(modules, clr_prefix + tail.moduleName, tail.pyHandle);
+                    Runtime.PyDict_SetItemString(modules, clr_prefix + tail.moduleName, tail.ObjectReference);
                 }
             }
 
@@ -380,7 +373,7 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw)
             }
         }
 
-        private static bool IsLoadAll(IntPtr fromList)
+        private static bool IsLoadAll(BorrowedReference fromList)
         {
             if (CLRModule.preload)
             {
@@ -390,10 +383,8 @@ private static bool IsLoadAll(IntPtr fromList)
             {
                 return false;
             }
-            IntPtr fp = Runtime.PySequence_GetItem(fromList, 0);
-            bool res = Runtime.GetManagedString(fp) == "*";
-            Runtime.XDecref(fp);
-            return res;
+            using var fp = Runtime.PySequence_GetItem(fromList, 0);
+            return Runtime.GetManagedString(fp) == "*";
         }
     }
 }