Improve geninterop script to handle new case in 3.11

pythonnet · filmor · Nov 2, 2022 · Sep 18, 2022 · Sep 18, 2022 · Sep 19, 2022
commit c0b4eb285ef3af1fdf90b0461e14c26b9969cf7a
diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py
@@ -13,40 +13,26 @@
     - clang
 """
 
-from __future__ import print_function
-
-import logging
 import os
+import shutil
 import sys
 import sysconfig
 import subprocess
 
-if sys.version_info.major > 2:
-    from io import StringIO
-else:
-    from StringIO import StringIO
-
+from io import StringIO
+from pathlib import Path
 from pycparser import c_ast, c_parser
 
-_log = logging.getLogger()
-logging.basicConfig(level=logging.DEBUG)
-
-PY_MAJOR = sys.version_info[0]
-PY_MINOR = sys.version_info[1]
-
 # rename some members from their C name when generating the C#
 _typeoffset_member_renames = {
     "ht_name": "name",
-    "ht_qualname": "qualname"
+    "ht_qualname": "qualname",
+    "getitem": "spec_cache_getitem",
 }
 
 
 def _check_output(*args, **kwargs):
-    """Check output wrapper for py2/py3 compatibility"""
-    output = subprocess.check_output(*args, **kwargs)
-    if PY_MAJOR == 2:
-        return output
-    return output.decode("ascii")
+    return subprocess.check_output(*args, **kwargs, encoding="utf8")
 
 
 class AstParser(object):
@@ -92,7 +78,7 @@ def visit(self, node):
             self.visit_identifier(node)
 
     def visit_ast(self, ast):
-        for name, node in ast.children():
+        for _name, node in ast.children():
             self.visit(node)
 
     def visit_typedef(self, typedef):
@@ -113,7 +99,7 @@ def visit_struct(self, struct):
                 self.visit(decl)
                 self._struct_members_stack.pop(0)
             self._struct_stack.pop(0)
-        elif self._ptr_decl_depth:
+        elif self._ptr_decl_depth or self._struct_members_stack:
             # the struct is empty, but add it as a member to the current
             # struct as the current member maybe a pointer to it.
             self._add_struct_member(struct.name)
@@ -141,7 +127,8 @@ def _add_struct_member(self, type_name):
         current_struct = self._struct_stack[0]
         member_name = self._struct_members_stack[0]
         struct_members = self._struct_members.setdefault(
-            self._get_struct_name(current_struct), [])
+            self._get_struct_name(current_struct), []
+        )
 
         # get the node associated with this type
         node = None
@@ -179,7 +166,6 @@ def _get_struct_name(self, node):
 
 
 class Writer(object):
-
     def __init__(self):
         self._stream = StringIO()
 
@@ -193,43 +179,56 @@ def to_string(self):
         return self._stream.getvalue()
 
 
-def preprocess_python_headers():
+def preprocess_python_headers(*, cc=None, include_py=None):
     """Return Python.h pre-processed, ready for parsing.
     Requires clang.
     """
-    fake_libc_include = os.path.join(os.path.dirname(__file__),
-                                     "fake_libc_include")
+    this_path = Path(__file__).parent
+
+    fake_libc_include = this_path / "fake_libc_include"
     include_dirs = [fake_libc_include]
 
-    include_py = sysconfig.get_config_var("INCLUDEPY")
+    if cc is None:
+        cc = shutil.which("clang")
+    if cc is None:
+        cc = shutil.which("gcc")
+    if cc is None:
+        raise RuntimeError("No suitable C compiler found, need clang or gcc")
+
+    if include_py is None:
+        include_py = sysconfig.get_config_var("INCLUDEPY")
+    include_py = Path(include_py)
+
     include_dirs.append(include_py)
 
-    include_args = [c for p in include_dirs for c in ["-I", p]]
+    include_args = [c for p in include_dirs for c in ["-I", str(p)]]
 
+    # fmt: off
     defines = [
         "-D", "__attribute__(x)=",
         "-D", "__inline__=inline",
         "-D", "__asm__=;#pragma asm",
         "-D", "__int64=long long",
-        "-D", "_POSIX_THREADS"
+        "-D", "_POSIX_THREADS",
     ]
 
-    if os.name == 'nt':
+    if sys.platform == "win32":
         defines.extend([
             "-D", "__inline=inline",
             "-D", "__ptr32=",
             "-D", "__ptr64=",
             "-D", "__declspec(x)=",
         ])
+    #fmt: on
 
     if hasattr(sys, "abiflags"):
         if "d" in sys.abiflags:
             defines.extend(("-D", "PYTHON_WITH_PYDEBUG"))
         if "u" in sys.abiflags:
             defines.extend(("-D", "PYTHON_WITH_WIDE_UNICODE"))
 
-    python_h = os.path.join(include_py, "Python.h")
-    cmd = ["clang", "-pthread"] + include_args + defines + ["-E", python_h]
+    python_h = include_py / "Python.h"
+    cmd = [cc, "-pthread"] + include_args + defines + ["-E", str(python_h)]
 
     # normalize as the parser doesn't like windows line endings.
     lines = []
@@ -240,16 +239,13 @@ def preprocess_python_headers():
     return "\n".join(lines)
 
 
-
-def gen_interop_head(writer):
+def gen_interop_head(writer, version, abi_flags):
     filename = os.path.basename(__file__)
-    abi_flags = getattr(sys, "abiflags", "").replace("m", "")
-    py_ver = "{0}.{1}".format(PY_MAJOR, PY_MINOR)
-    class_definition = """
-// Auto-generated by %s.
+    class_definition = f"""
+// Auto-generated by {filename}.
 // DO NOT MODIFY BY HAND.
 
-// Python %s: ABI flags: '%s'
+// Python {".".join(version[:2])}: ABI flags: '{abi_flags}'
 
 // ReSharper disable InconsistentNaming
 // ReSharper disable IdentifierTypo
@@ -261,7 +257,7 @@ def gen_interop_head(writer):
 using Python.Runtime.Native;
 
 namespace Python.Runtime
-{""" % (filename, py_ver, abi_flags)
+{{"""
     writer.extend(class_definition)
 
 
@@ -271,25 +267,24 @@ def gen_interop_tail(writer):
     writer.extend(tail)
 
 
-def gen_heap_type_members(parser, writer, type_name = None):
+def gen_heap_type_members(parser, writer, type_name):
     """Generate the TypeOffset C# class"""
     members = parser.get_struct_members("PyHeapTypeObject")
-    type_name = type_name or "TypeOffset{0}{1}".format(PY_MAJOR, PY_MINOR)
-    class_definition = """
+    class_definition = f"""
     [SuppressMessage("Style", "IDE1006:Naming Styles",
                      Justification = "Following CPython",
                      Scope = "type")]
 
     [StructLayout(LayoutKind.Sequential)]
-    internal class {0} : GeneratedTypeOffsets, ITypeOffsets
+    internal class {type_name} : GeneratedTypeOffsets, ITypeOffsets
     {{
-        public {0}() {{ }}
+        public {type_name}() {{ }}
         // Auto-generated from PyHeapTypeObject in Python.h
-""".format(type_name)
+"""
 
     # All the members are sizeof(void*) so we don't need to do any
     # extra work to determine the size based on the type.
-    for name, tpy in members:
+    for name, _type in members:
         name = _typeoffset_member_renames.get(name, name)
         class_definition += "        public int %s  { get; private set; }\n" % name
 
@@ -304,17 +299,18 @@ def gen_structure_code(parser, writer, type_name, indent):
         return False
     out = writer.append
     out(indent, "[StructLayout(LayoutKind.Sequential)]")
-    out(indent, "internal struct %s" % type_name)
+    out(indent, f"internal struct {type_name}")
     out(indent, "{")
-    for name, tpy in members:
-        out(indent + 1, "public IntPtr %s;" % name)
+    for name, _type in members:
+        out(indent + 1, f"public IntPtr {name};")
     out(indent, "}")
     out()
     return True
 
-def main():
+
+def main(*, cc=None, include_py=None, version=None, out=None):
     # preprocess Python.h and build the AST
-    python_h = preprocess_python_headers()
+    python_h = preprocess_python_headers(cc=cc, include_py=include_py)
     parser = c_parser.CParser()
     ast = parser.parse(python_h)
 
@@ -323,21 +319,47 @@ def main():
     ast_parser.visit(ast)
 
     writer = Writer()
+
+    if include_py and not version:
+        raise RuntimeError("If the include path is overridden, version must be "
+                "defined"
+                )
+
+    if version:
+        version = version.split('.')
+    else:
+        version = sys.version_info
+
     # generate the C# code
-    offsets_type_name = "NativeTypeOffset" if len(sys.argv) > 1 else None
-    gen_interop_head(writer)
+    abi_flags = getattr(sys, "abiflags", "").replace("m", "")
+    gen_interop_head(writer, version, abi_flags)
 
-    gen_heap_type_members(ast_parser, writer, type_name = offsets_type_name)
+    type_name = f"TypeOffset{version[0]}{version[1]}{abi_flags}"
+    gen_heap_type_members(ast_parser, writer, type_name)
 
     gen_interop_tail(writer)
 
     interop_cs = writer.to_string()
-    if len(sys.argv) > 1:
-        with open(sys.argv[1], "w") as fh:
-            fh.write(interop_cs)
-    else:
+    if not out or out == "-":
         print(interop_cs)
+    else:
+        with open(out, "w") as fh:
+            fh.write(interop_cs)
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    import argparse
+
+    a = argparse.ArgumentParser("Interop file generator for Python.NET")
+    a.add_argument("--cc", help="C compiler to use, either clang or gcc")
+    a.add_argument("--include-py", help="Include path of Python")
+    a.add_argument("--version", help="Python version")
+    a.add_argument("--out", help="Output path", default="-")
+    args = a.parse_args()
+
+    sys.exit(main(
+        cc=args.cc,
+        include_py=args.include_py,
+        out=args.out,
+        version=args.version
+        ))