diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 21402a42e..5b782c8b4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -24,7 +24,7 @@ jobs: path: doc/build/html/ deploy: - if: github.ref == 'refs/heads/release' + if: github.ref == 'refs/heads/master' runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2cc793621..93963c70a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: fail-fast: false matrix: os: [windows, ubuntu, macos] - python: ["3.7", "3.8", "3.9", "3.10"] + python: ["3.7", "3.8", "3.9", "3.10", "3.11"] platform: [x64, x86] exclude: - os: ubuntu @@ -54,15 +54,17 @@ jobs: run: | pip install -v . - - name: Set Python DLL path (non Windows) + - name: Set Python DLL path and PYTHONHOME (non Windows) if: ${{ matrix.os != 'windows' }} run: | echo PYTHONNET_PYDLL=$(python -m find_libpython) >> $GITHUB_ENV + echo PYTHONHOME=$(python -c 'import sys; print(sys.prefix)') >> $GITHUB_ENV - - name: Set Python DLL path (Windows) + - name: Set Python DLL path and PYTHONHOME (Windows) if: ${{ matrix.os == 'windows' }} run: | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -InputObject "PYTHONNET_PYDLL=$(python -m find_libpython)" + Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -InputObject "PYTHONHOME=$(python -c 'import sys; print(sys.prefix)')" - name: Embedding tests run: dotnet test --runtime any-${{ matrix.platform }} --logger "console;verbosity=detailed" src/embed_tests/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f0f212e6..13bf09c2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,31 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added +### Changed + +### Fixed + +## [3.0.1](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.1) - 2022-11-03 + +### Added + +- Support for Python 3.11 + +### Changed + +- Allow decoders to override conversion of types derived from primitive types + +### Fixed + +- Fixed objects leaking when Python attached event handlers to them even if they were later removed +- Fixed `PyInt` conversion to `BigInteger` and `System.String` produced incorrect result for values between 128 and 255. +- Fixed implementing a generic interface with a Python class + + +## [3.0.0](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.0) - 2022-09-29 + +### Added + - Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax - Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). - Add GetPythonThreadID and Interrupt methods in PythonEngine @@ -134,7 +159,24 @@ There is no need to specify it. - support for .NET Framework 4.0-4.6; Mono before 5.4. Python.NET now requires .NET Standard 2.0 (see [the matrix](https://docs.microsoft.com/en-us/dotnet/standard/net-standard#net-implementation-support)) -## [2.5.0][] - 2020-06-14 +## [2.5.2](https://github.com/pythonnet/pythonnet/releases/tag/v2.5.2) - 2021-02-05 + +Bugfix release. + +### Fixed +- Fix `object[]` parameters taking precedence when should not in overload resolution +- Empty parameter names (as can be generated from F#) do not cause crashes + +## [2.5.1](https://github.com/pythonnet/pythonnet/releases/tag/v2.5.1) - 2020-06-18 + +Bugfix release. + +### Fixed + +- Fix incorrect dereference of wrapper object in `tp_repr`, which may result in a program crash +- Fix incorrect dereference in params array handling + +## [2.5.0](https://github.com/pythonnet/pythonnet/releases/tag/v2.5.0) - 2020-06-14 This version improves performance on benchmarks significantly compared to 2.3. diff --git a/pyproject.toml b/pyproject.toml index 39c7c14fb..52f1adb18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,8 @@ dependencies = [ "clr_loader>=0.2.2,<0.3.0" ] +requires-python = ">=3.7, <3.12" + classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", @@ -23,6 +25,7 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", "Operating System :: MacOS :: MacOS X", diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index be416bc15..9b764d43f 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -355,6 +355,19 @@ from datetime import datetime scope.Exec("Codecs.AcceptsDateTime(datetime(2021, 1, 22))"); } + [Test] + public void FloatDerivedDecoded() + { + using var scope = Py.CreateScope(); + scope.Exec(@"class FloatDerived(float): pass"); + using var floatDerived = scope.Eval("FloatDerived"); + var decoder = new DecoderReturningPredefinedValue(floatDerived, 42); + PyObjectConversions.RegisterDecoder(decoder); + using var result = scope.Eval("FloatDerived()"); + object decoded = result.As(); + Assert.AreEqual(42, decoded); + } + [Test] public void ExceptionDecodedNoInstance() { diff --git a/src/embed_tests/Events.cs b/src/embed_tests/Events.cs new file mode 100644 index 000000000..c216f4214 --- /dev/null +++ b/src/embed_tests/Events.cs @@ -0,0 +1,67 @@ +using System; +using System.Diagnostics; +using System.Threading; + +using NUnit.Framework; + +using Python.Runtime; + +namespace Python.EmbeddingTest; + +public class Events +{ + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void UsingDoesNotLeak() + { + using var scope = Py.CreateScope(); + scope.Exec(@" +import gc + +from Python.EmbeddingTest import ClassWithEventHandler + +def event_handler(): + pass + +for _ in range(2000): + example = ClassWithEventHandler() + example.LeakEvent += event_handler + example.LeakEvent -= event_handler + del example + +gc.collect() +"); + Runtime.Runtime.TryCollectingGarbage(10); + Assert.AreEqual(0, ClassWithEventHandler.alive); + } +} + +public class ClassWithEventHandler +{ + internal static int alive; + + public event EventHandler LeakEvent; + private Array arr; // dummy array to exacerbate memory leak + + public ClassWithEventHandler() + { + Interlocked.Increment(ref alive); + this.arr = new int[800]; + } + + ~ClassWithEventHandler() + { + Interlocked.Decrement(ref alive); + } +} diff --git a/src/embed_tests/TestConverter.cs b/src/embed_tests/TestConverter.cs index e586eda1b..0686d528b 100644 --- a/src/embed_tests/TestConverter.cs +++ b/src/embed_tests/TestConverter.cs @@ -148,7 +148,7 @@ public void PyIntImplicit() { var i = new PyInt(1); var ni = (PyObject)i.As(); - Assert.AreEqual(i.rawPtr, ni.rawPtr); + Assert.IsTrue(PythonReferenceComparer.Instance.Equals(i, ni)); } [Test] @@ -178,8 +178,11 @@ public void RawPyObjectProxy() var clrObject = (CLRObject)ManagedType.GetManagedObject(pyObjectProxy); Assert.AreSame(pyObject, clrObject.inst); - var proxiedHandle = pyObjectProxy.GetAttr("Handle").As(); - Assert.AreEqual(pyObject.Handle, proxiedHandle); +#pragma warning disable CS0612 // Type or member is obsolete + const string handlePropertyName = nameof(PyObject.Handle); +#pragma warning restore CS0612 // Type or member is obsolete + var proxiedHandle = pyObjectProxy.GetAttr(handlePropertyName).As(); + Assert.AreEqual(pyObject.DangerousGetAddressOrNull(), proxiedHandle); } // regression for https://github.com/pythonnet/pythonnet/issues/451 diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 498119d1e..a0f9b63eb 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -99,8 +99,7 @@ from Python.EmbeddingTest.Domain import MyClass { Debug.Assert(obj.AsManagedObject(type).GetType() == type); // We only needs its Python handle - PyRuntime.XIncref(obj); - return obj.Handle; + return new NewReference(obj).DangerousMoveToPointer(); } } } diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 40ab03395..b748a2244 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -212,7 +212,9 @@ public void ValidateRefCount() Assert.AreEqual(ptr, e.Handle); Assert.AreEqual(2, e.ImpactedObjects.Count); // Fix for this test, don't do this on general environment +#pragma warning disable CS0618 // Type or member is obsolete Runtime.Runtime.XIncref(e.Reference); +#pragma warning restore CS0618 // Type or member is obsolete return false; }; Finalizer.Instance.IncorrectRefCntResolver += handler; @@ -234,8 +236,9 @@ private static IntPtr CreateStringGarbage() { PyString s1 = new PyString("test_string"); // s2 steal a reference from s1 - PyString s2 = new PyString(StolenReference.DangerousFromPointer(s1.Handle)); - return s1.Handle; + IntPtr address = s1.Reference.DangerousGetAddress(); + PyString s2 = new (StolenReference.DangerousFromPointer(address)); + return address; } } } diff --git a/src/embed_tests/TestNativeTypeOffset.cs b/src/embed_tests/TestNativeTypeOffset.cs index 2d31fe506..d692c24e6 100644 --- a/src/embed_tests/TestNativeTypeOffset.cs +++ b/src/embed_tests/TestNativeTypeOffset.cs @@ -33,7 +33,8 @@ public void LoadNativeTypeOffsetClass() { PyObject sys = Py.Import("sys"); // We can safely ignore the "m" abi flag - var abiflags = sys.GetAttr("abiflags", "".ToPython()).ToString().Replace("m", ""); + var abiflags = sys.HasAttr("abiflags") ? sys.GetAttr("abiflags").ToString() : ""; + abiflags = abiflags.Replace("m", ""); if (!string.IsNullOrEmpty(abiflags)) { string typeName = "Python.Runtime.NativeTypeOffset, Python.Runtime"; diff --git a/src/embed_tests/TestPyInt.cs b/src/embed_tests/TestPyInt.cs index 822fe0715..c147e074b 100644 --- a/src/embed_tests/TestPyInt.cs +++ b/src/embed_tests/TestPyInt.cs @@ -191,16 +191,23 @@ public void ToBigInteger() { 0, 1, 2, 0x10, + 0x79, + 0x80, + 0x81, + 0xFF, 0x123, + 0x8000, 0x1234, + 0x8001, + 0x4000, + 0xFF, }; simpleValues = simpleValues.Concat(simpleValues.Select(v => -v)).ToArray(); - foreach (var val in simpleValues) - { - var pyInt = new PyInt(val); - Assert.AreEqual((BigInteger)val, pyInt.ToBigInteger()); - } + var expected = simpleValues.Select(v => new BigInteger(v)).ToArray(); + var actual = simpleValues.Select(v => new PyInt(v).ToBigInteger()).ToArray(); + + CollectionAssert.AreEqual(expected, actual); } [Test] diff --git a/src/embed_tests/TestPythonEngineProperties.cs b/src/embed_tests/TestPythonEngineProperties.cs index ca9164a1d..be91d7f45 100644 --- a/src/embed_tests/TestPythonEngineProperties.cs +++ b/src/embed_tests/TestPythonEngineProperties.cs @@ -9,6 +9,7 @@ public class TestPythonEngineProperties [Test] public static void GetBuildinfoDoesntCrash() { + PythonEngine.Initialize(); using (Py.GIL()) { string s = PythonEngine.BuildInfo; @@ -21,6 +22,7 @@ public static void GetBuildinfoDoesntCrash() [Test] public static void GetCompilerDoesntCrash() { + PythonEngine.Initialize(); using (Py.GIL()) { string s = PythonEngine.Compiler; @@ -34,6 +36,7 @@ public static void GetCompilerDoesntCrash() [Test] public static void GetCopyrightDoesntCrash() { + PythonEngine.Initialize(); using (Py.GIL()) { string s = PythonEngine.Copyright; @@ -46,6 +49,7 @@ public static void GetCopyrightDoesntCrash() [Test] public static void GetPlatformDoesntCrash() { + PythonEngine.Initialize(); using (Py.GIL()) { string s = PythonEngine.Platform; @@ -58,6 +62,7 @@ public static void GetPlatformDoesntCrash() [Test] public static void GetVersionDoesntCrash() { + PythonEngine.Initialize(); using (Py.GIL()) { string s = PythonEngine.Version; @@ -91,9 +96,6 @@ public static void GetProgramNameDefault() /// Test default behavior of PYTHONHOME. If ENVVAR is set it will /// return the same value. If not, returns EmptyString. /// - /// - /// AppVeyor.yml has been update to tests with ENVVAR set. - /// [Test] public static void GetPythonHomeDefault() { @@ -109,22 +111,19 @@ public static void GetPythonHomeDefault() [Test] public void SetPythonHome() { - // We needs to ensure that engine was started and shutdown at least once before setting dummy home. - // Otherwise engine will not run with dummy path with random problem. - if (!PythonEngine.IsInitialized) - { - PythonEngine.Initialize(); - } - + PythonEngine.Initialize(); + var pythonHomeBackup = PythonEngine.PythonHome; PythonEngine.Shutdown(); - var pythonHomeBackup = PythonEngine.PythonHome; + if (pythonHomeBackup == "") + Assert.Inconclusive("Can't reset PythonHome to empty string, skipping"); var pythonHome = "/dummypath/"; PythonEngine.PythonHome = pythonHome; PythonEngine.Initialize(); + Assert.AreEqual(pythonHome, PythonEngine.PythonHome); PythonEngine.Shutdown(); // Restoring valid pythonhome. @@ -134,15 +133,12 @@ public void SetPythonHome() [Test] public void SetPythonHomeTwice() { - // We needs to ensure that engine was started and shutdown at least once before setting dummy home. - // Otherwise engine will not run with dummy path with random problem. - if (!PythonEngine.IsInitialized) - { - PythonEngine.Initialize(); - } + PythonEngine.Initialize(); + var pythonHomeBackup = PythonEngine.PythonHome; PythonEngine.Shutdown(); - var pythonHomeBackup = PythonEngine.PythonHome; + if (pythonHomeBackup == "") + Assert.Inconclusive("Can't reset PythonHome to empty string, skipping"); var pythonHome = "/dummypath/"; @@ -156,6 +152,26 @@ public void SetPythonHomeTwice() PythonEngine.PythonHome = pythonHomeBackup; } + [Test] + [Ignore("Currently buggy in Python")] + public void SetPythonHomeEmptyString() + { + PythonEngine.Initialize(); + + var backup = PythonEngine.PythonHome; + if (backup == "") + { + PythonEngine.Shutdown(); + Assert.Inconclusive("Can't reset PythonHome to empty string, skipping"); + } + PythonEngine.PythonHome = ""; + + Assert.AreEqual("", PythonEngine.PythonHome); + + PythonEngine.PythonHome = backup; + PythonEngine.Shutdown(); + } + [Test] public void SetProgramName() { @@ -202,7 +218,7 @@ public void SetPythonPath() // The list sys.path is initialized with this value on interpreter startup; // it can be (and usually is) modified later to change the search path for loading modules. // See https://docs.python.org/3/c-api/init.html#c.Py_GetPath - // After PythonPath is set, then PythonEngine.PythonPath will correctly return the full search path. + // After PythonPath is set, then PythonEngine.PythonPath will correctly return the full search path. PythonEngine.Shutdown(); diff --git a/src/embed_tests/TestPythonException.cs b/src/embed_tests/TestPythonException.cs index a7cf05c83..a248b6a1f 100644 --- a/src/embed_tests/TestPythonException.cs +++ b/src/embed_tests/TestPythonException.cs @@ -161,7 +161,7 @@ def __init__(self, val): using var tbObj = tbPtr.MoveToPyObject(); // the type returned from PyErr_NormalizeException should not be the same type since a new // exception was raised by initializing the exception - Assert.AreNotEqual(type.Handle, typeObj.Handle); + Assert.IsFalse(PythonReferenceComparer.Instance.Equals(type, typeObj)); // the message should now be the string from the throw exception during normalization Assert.AreEqual("invalid literal for int() with base 10: 'dummy string'", strObj.ToString()); } diff --git a/src/runtime/Converter.cs b/src/runtime/Converter.cs index 3c46e9034..73bbd4a3a 100644 --- a/src/runtime/Converter.cs +++ b/src/runtime/Converter.cs @@ -361,28 +361,44 @@ internal static bool ToManagedValue(BorrowedReference value, Type obType, // conversions (Python string -> managed string). if (obType == objectType) { - if (Runtime.PyString_Check(value)) + if (Runtime.PyString_CheckExact(value)) { return ToPrimitive(value, stringType, out result, setError); } - if (Runtime.PyBool_Check(value)) + if (Runtime.PyBool_CheckExact(value)) { return ToPrimitive(value, boolType, out result, setError); } - if (Runtime.PyFloat_Check(value)) + if (Runtime.PyFloat_CheckExact(value)) { return ToPrimitive(value, doubleType, out result, setError); } - // give custom codecs a chance to take over conversion of ints and sequences + // give custom codecs a chance to take over conversion + // of ints, sequences, and types derived from primitives BorrowedReference pyType = Runtime.PyObject_TYPE(value); if (PyObjectConversions.TryDecode(value, pyType, obType, out result)) { return true; } + if (Runtime.PyString_Check(value)) + { + return ToPrimitive(value, stringType, out result, setError); + } + + if (Runtime.PyBool_Check(value)) + { + return ToPrimitive(value, boolType, out result, setError); + } + + if (Runtime.PyFloat_Check(value)) + { + return ToPrimitive(value, doubleType, out result, setError); + } + if (Runtime.PyInt_Check(value)) { result = new PyInt(value); diff --git a/src/runtime/Finalizer.cs b/src/runtime/Finalizer.cs index f4b465ecb..713564f08 100644 --- a/src/runtime/Finalizer.cs +++ b/src/runtime/Finalizer.cs @@ -364,6 +364,8 @@ struct PendingFinalization { public IntPtr PyObj; public BorrowedReference Ref => new(PyObj); + public ManagedType? Managed => ManagedType.GetManagedObject(Ref); + public nint RefCount => Runtime.Refcount(Ref); public int RuntimeRun; #if TRACE_ALLOC public string StackTrace; diff --git a/src/runtime/InternString.cs b/src/runtime/InternString.cs index b6d9a0e4a..decb3981d 100644 --- a/src/runtime/InternString.cs +++ b/src/runtime/InternString.cs @@ -42,7 +42,7 @@ public static void Initialize() Debug.Assert(name == op.As()); SetIntern(name, op); var field = type.GetField("f" + name, PyIdentifierFieldFlags)!; - field.SetValue(null, op.rawPtr); + field.SetValue(null, op.DangerousGetAddressOrNull()); } } @@ -76,7 +76,7 @@ public static bool TryGetInterned(BorrowedReference op, out string s) private static void SetIntern(string s, PyString op) { _string2interns.Add(s, op); - _intern2strings.Add(op.rawPtr, s); + _intern2strings.Add(op.Reference.DangerousGetAddress(), s); } } } diff --git a/src/runtime/Native/TypeOffset311.cs b/src/runtime/Native/TypeOffset311.cs new file mode 100644 index 000000000..de5afacb9 --- /dev/null +++ b/src/runtime/Native/TypeOffset311.cs @@ -0,0 +1,141 @@ + +// Auto-generated by geninterop.py. +// DO NOT MODIFY BY HAND. + +// Python 3.11: ABI flags: '' + +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + + [StructLayout(LayoutKind.Sequential)] + internal class TypeOffset311 : GeneratedTypeOffsets, ITypeOffsets + { + public TypeOffset311() { } + // Auto-generated from PyHeapTypeObject in Python.h + public int ob_refcnt { get; private set; } + public int ob_type { get; private set; } + public int ob_size { get; private set; } + public int tp_name { get; private set; } + public int tp_basicsize { get; private set; } + public int tp_itemsize { get; private set; } + public int tp_dealloc { get; private set; } + public int tp_vectorcall_offset { get; private set; } + public int tp_getattr { get; private set; } + public int tp_setattr { get; private set; } + public int tp_as_async { get; private set; } + public int tp_repr { get; private set; } + public int tp_as_number { get; private set; } + public int tp_as_sequence { get; private set; } + public int tp_as_mapping { get; private set; } + public int tp_hash { get; private set; } + public int tp_call { get; private set; } + public int tp_str { get; private set; } + public int tp_getattro { get; private set; } + public int tp_setattro { get; private set; } + public int tp_as_buffer { get; private set; } + public int tp_flags { get; private set; } + public int tp_doc { get; private set; } + public int tp_traverse { get; private set; } + public int tp_clear { get; private set; } + public int tp_richcompare { get; private set; } + public int tp_weaklistoffset { get; private set; } + public int tp_iter { get; private set; } + public int tp_iternext { get; private set; } + public int tp_methods { get; private set; } + public int tp_members { get; private set; } + public int tp_getset { get; private set; } + public int tp_base { get; private set; } + public int tp_dict { get; private set; } + public int tp_descr_get { get; private set; } + public int tp_descr_set { get; private set; } + public int tp_dictoffset { get; private set; } + public int tp_init { get; private set; } + public int tp_alloc { get; private set; } + public int tp_new { get; private set; } + public int tp_free { get; private set; } + public int tp_is_gc { get; private set; } + public int tp_bases { get; private set; } + public int tp_mro { get; private set; } + public int tp_cache { get; private set; } + public int tp_subclasses { get; private set; } + public int tp_weaklist { get; private set; } + public int tp_del { get; private set; } + public int tp_version_tag { get; private set; } + public int tp_finalize { get; private set; } + public int tp_vectorcall { get; private set; } + public int am_await { get; private set; } + public int am_aiter { get; private set; } + public int am_anext { get; private set; } + public int am_send { get; private set; } + public int nb_add { get; private set; } + public int nb_subtract { get; private set; } + public int nb_multiply { get; private set; } + public int nb_remainder { get; private set; } + public int nb_divmod { get; private set; } + public int nb_power { get; private set; } + public int nb_negative { get; private set; } + public int nb_positive { get; private set; } + public int nb_absolute { get; private set; } + public int nb_bool { get; private set; } + public int nb_invert { get; private set; } + public int nb_lshift { get; private set; } + public int nb_rshift { get; private set; } + public int nb_and { get; private set; } + public int nb_xor { get; private set; } + public int nb_or { get; private set; } + public int nb_int { get; private set; } + public int nb_reserved { get; private set; } + public int nb_float { get; private set; } + public int nb_inplace_add { get; private set; } + public int nb_inplace_subtract { get; private set; } + public int nb_inplace_multiply { get; private set; } + public int nb_inplace_remainder { get; private set; } + public int nb_inplace_power { get; private set; } + public int nb_inplace_lshift { get; private set; } + public int nb_inplace_rshift { get; private set; } + public int nb_inplace_and { get; private set; } + public int nb_inplace_xor { get; private set; } + public int nb_inplace_or { get; private set; } + public int nb_floor_divide { get; private set; } + public int nb_true_divide { get; private set; } + public int nb_inplace_floor_divide { get; private set; } + public int nb_inplace_true_divide { get; private set; } + public int nb_index { get; private set; } + public int nb_matrix_multiply { get; private set; } + public int nb_inplace_matrix_multiply { get; private set; } + public int mp_length { get; private set; } + public int mp_subscript { get; private set; } + public int mp_ass_subscript { get; private set; } + public int sq_length { get; private set; } + public int sq_concat { get; private set; } + public int sq_repeat { get; private set; } + public int sq_item { get; private set; } + public int was_sq_slice { get; private set; } + public int sq_ass_item { get; private set; } + public int was_sq_ass_slice { get; private set; } + public int sq_contains { get; private set; } + public int sq_inplace_concat { get; private set; } + public int sq_inplace_repeat { get; private set; } + public int bf_getbuffer { get; private set; } + public int bf_releasebuffer { get; private set; } + public int name { get; private set; } + public int ht_slots { get; private set; } + public int qualname { get; private set; } + public int ht_cached_keys { get; private set; } + public int ht_module { get; private set; } + public int _ht_tpname { get; private set; } + public int spec_cache_getitem { get; private set; } + } +} diff --git a/src/runtime/PythonEngine.cs b/src/runtime/PythonEngine.cs index e5879ae67..4ed45b9e9 100644 --- a/src/runtime/PythonEngine.cs +++ b/src/runtime/PythonEngine.cs @@ -47,6 +47,14 @@ public static bool IsInitialized get { return initialized; } } + private static void EnsureInitialized() + { + if (!IsInitialized) + throw new InvalidOperationException( + "Python must be initialized for this operation" + ); + } + /// Set to true to enable GIL debugging assistance. public static bool DebugGIL { get; set; } = false; @@ -96,6 +104,7 @@ public static string PythonHome { get { + EnsureInitialized(); IntPtr p = Runtime.TryUsingDll(() => Runtime.Py_GetPythonHome()); return UcsMarshaler.PtrToPy3UnicodePy2String(p) ?? ""; } @@ -103,10 +112,8 @@ public static string PythonHome { // this value is null in the beginning Marshal.FreeHGlobal(_pythonHome); - _pythonHome = Runtime.TryUsingDll( - () => UcsMarshaler.Py3UnicodePy2StringtoPtr(value) - ); - Runtime.Py_SetPythonHome(_pythonHome); + _pythonHome = UcsMarshaler.Py3UnicodePy2StringtoPtr(value); + Runtime.TryUsingDll(() => Runtime.Py_SetPythonHome(_pythonHome)); } } @@ -128,7 +135,7 @@ public static string PythonPath } public static Version MinSupportedVersion => new(3, 7); - public static Version MaxSupportedVersion => new(3, 10, int.MaxValue, int.MaxValue); + public static Version MaxSupportedVersion => new(3, 11, int.MaxValue, int.MaxValue); public static bool IsSupportedVersion(Version version) => version >= MinSupportedVersion && version <= MaxSupportedVersion; public static string Version diff --git a/src/runtime/PythonTypes/PyInt.cs b/src/runtime/PythonTypes/PyInt.cs index 6b3dbf210..e71462b74 100644 --- a/src/runtime/PythonTypes/PyInt.cs +++ b/src/runtime/PythonTypes/PyInt.cs @@ -212,7 +212,7 @@ public BigInteger ToBigInteger() offset++; neg = true; } - byte[] littleEndianBytes = new byte[(hex.Length - offset + 1) / 2]; + byte[] littleEndianBytes = new byte[(hex.Length - offset + 1) / 2 + 1]; for (; offset < hex.Length; offset++) { int littleEndianHexIndex = hex.Length - 1 - offset; diff --git a/src/runtime/PythonTypes/PyObject.cs b/src/runtime/PythonTypes/PyObject.cs index 3d48e22ed..bda2d9c02 100644 --- a/src/runtime/PythonTypes/PyObject.cs +++ b/src/runtime/PythonTypes/PyObject.cs @@ -27,7 +27,7 @@ public partial class PyObject : DynamicObject, IDisposable, ISerializable public StackTrace Traceback { get; } = new StackTrace(1); #endif - protected internal IntPtr rawPtr = IntPtr.Zero; + protected IntPtr rawPtr = IntPtr.Zero; internal readonly int run = Runtime.GetRun(); internal BorrowedReference obj => new (rawPtr); @@ -252,6 +252,8 @@ internal void Leak() rawPtr = IntPtr.Zero; } + internal IntPtr DangerousGetAddressOrNull() => rawPtr; + internal void CheckRun() { if (run != Runtime.GetRun()) @@ -1049,9 +1051,20 @@ public PyList Dir() return Runtime.GetManagedString(strval.BorrowOrThrow()); } - string? DebuggerDisplay => DebugUtil.HaveInterpreterLock() - ? this.ToString() - : $"pyobj at 0x{this.rawPtr:X} (get Py.GIL to see more info)"; + ManagedType? InternalManagedObject => ManagedType.GetManagedObject(this.Reference); + + string? DebuggerDisplay + { + get + { + if (DebugUtil.HaveInterpreterLock()) + return this.ToString(); + var obj = this.InternalManagedObject; + return obj is { } + ? obj.ToString() + : $"pyobj at 0x{this.rawPtr:X} (get Py.GIL to see more info)"; + } + } /// diff --git a/src/runtime/PythonTypes/PyType.cs b/src/runtime/PythonTypes/PyType.cs index 260800592..af796a5c5 100644 --- a/src/runtime/PythonTypes/PyType.cs +++ b/src/runtime/PythonTypes/PyType.cs @@ -155,6 +155,7 @@ private static StolenReference FromSpec(TypeSpec spec, PyTuple? bases = null) using var nativeSpec = new NativeTypeSpec(spec); var basesRef = bases is null ? default : bases.Reference; var result = Runtime.PyType_FromSpecWithBases(in nativeSpec, basesRef); + // Runtime.PyErr_Print(); return result.StealOrThrow(); } } diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 6238119ff..beb577e45 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -672,6 +672,9 @@ internal static unsafe nint Refcount(BorrowedReference op) [Pure] internal static int Refcount32(BorrowedReference op) => checked((int)Refcount(op)); + internal static void TryUsingDll(Action op) => + TryUsingDll(() => { op(); return 0; }); + /// /// Call specified function, and handle PythonDLL-related failures. /// @@ -1094,8 +1097,13 @@ internal static nint PyBuffer_SizeFromFormat(string format) internal static bool PyInt_Check(BorrowedReference ob) => PyObject_TypeCheck(ob, PyLongType); + internal static bool PyInt_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyLongType); + internal static bool PyBool_Check(BorrowedReference ob) => PyObject_TypeCheck(ob, PyBoolType); + internal static bool PyBool_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyBoolType); internal static NewReference PyInt_FromInt32(int value) => PyLong_FromLongLong(value); @@ -1141,6 +1149,8 @@ internal static NewReference PyLong_FromString(string value, int radix) internal static bool PyFloat_Check(BorrowedReference ob) => PyObject_TypeCheck(ob, PyFloatType); + internal static bool PyFloat_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyFloatType); /// /// Return value: New reference. @@ -1282,9 +1292,9 @@ internal static bool PyFloat_Check(BorrowedReference ob) // Python string API //==================================================================== internal static bool PyString_Check(BorrowedReference ob) - { - return PyObject_TYPE(ob) == PyStringType; - } + => PyObject_TypeCheck(ob, PyStringType); + internal static bool PyString_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyStringType); internal static NewReference PyString_FromString(string value) { @@ -1643,6 +1653,8 @@ internal static bool PyType_IsSubtype(BorrowedReference t1, BorrowedReference t2 return Delegates.PyType_IsSubtype(t1, t2); } + internal static bool PyObject_TypeCheckExact(BorrowedReference ob, BorrowedReference tp) + => PyObject_TYPE(ob) == tp; internal static bool PyObject_TypeCheck(BorrowedReference ob, BorrowedReference tp) { BorrowedReference t = PyObject_TYPE(ob); diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 217b4820e..e0a78ba49 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -475,17 +475,20 @@ internal static PyType CreateMetatypeWithGCHandleOffset() int size = Util.ReadInt32(Runtime.PyTypeType, TypeOffset.tp_basicsize) + IntPtr.Size // tp_clr_inst_offset ; - var result = new PyType(new TypeSpec("clr._internal.GCOffsetBase", basicSize: size, - new TypeSpec.Slot[] - { - - }, - TypeFlags.Default | TypeFlags.HeapType | TypeFlags.HaveGC), - bases: new PyTuple(new[] { py_type })); - - SetRequiredSlots(result, seen: new HashSet()); - Runtime.PyType_Modified(result); + var slots = new[] { + new TypeSpec.Slot(TypeSlotID.tp_traverse, subtype_traverse), + new TypeSpec.Slot(TypeSlotID.tp_clear, subtype_clear) + }; + var result = new PyType( + new TypeSpec( + "clr._internal.GCOffsetBase", + basicSize: size, + slots: slots, + TypeFlags.Default | TypeFlags.HeapType | TypeFlags.HaveGC + ), + bases: new PyTuple(new[] { py_type }) + ); return result; } @@ -617,6 +620,11 @@ internal static PyType AllocateTypeObject(string name, PyType metatype) Util.WriteRef(type, TypeOffset.name, new NewReference(temp).Steal()); Util.WriteRef(type, TypeOffset.qualname, temp.Steal()); + // Ensure that tp_traverse and tp_clear are always set, since their + // existence is enforced in newer Python versions in PyType_Ready + Util.WriteIntPtr(type, TypeOffset.tp_traverse, subtype_traverse); + Util.WriteIntPtr(type, TypeOffset.tp_clear, subtype_clear); + InheritSubstructs(type.Reference.DangerousGetAddress()); return type; diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 02288faee..cf6d9b16b 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -436,6 +436,7 @@ private static void AddVirtualMethod(MethodInfo method, Type baseType, TypeBuild il.Emit(OpCodes.Ldloc_0); il.Emit(OpCodes.Ldtoken, method); + il.Emit(OpCodes.Ldtoken, method.DeclaringType); #pragma warning disable CS0618 // PythonDerivedType is for internal use only if (method.ReturnType == typeof(void)) { @@ -505,6 +506,7 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde il.DeclareLocal(typeof(object[])); il.DeclareLocal(typeof(RuntimeMethodHandle)); + il.DeclareLocal(typeof(RuntimeTypeHandle)); // this il.Emit(OpCodes.Ldarg_0); @@ -546,6 +548,11 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde il.Emit(OpCodes.Ldloca_S, 1); il.Emit(OpCodes.Initobj, typeof(RuntimeMethodHandle)); il.Emit(OpCodes.Ldloc_1); + + // type handle is also not required + il.Emit(OpCodes.Ldloca_S, 2); + il.Emit(OpCodes.Initobj, typeof(RuntimeTypeHandle)); + il.Emit(OpCodes.Ldloc_2); #pragma warning disable CS0618 // PythonDerivedType is for internal use only // invoke the method @@ -698,7 +705,7 @@ public class PythonDerivedType /// class) it calls it, otherwise it calls the base method. /// public static T? InvokeMethod(IPythonDerivedType obj, string methodName, string origMethodName, - object[] args, RuntimeMethodHandle methodHandle) + object[] args, RuntimeMethodHandle methodHandle, RuntimeTypeHandle declaringTypeHandle) { var self = GetPyObj(obj); @@ -724,7 +731,10 @@ public class PythonDerivedType } PyObject py_result = method.Invoke(pyargs); - PyTuple? result_tuple = MarshalByRefsBack(args, methodHandle, py_result, outsOffset: 1); + var clrMethod = methodHandle != default + ? MethodBase.GetMethodFromHandle(methodHandle, declaringTypeHandle) + : null; + PyTuple? result_tuple = MarshalByRefsBack(args, clrMethod, py_result, outsOffset: 1); return result_tuple is not null ? result_tuple[0].As() : py_result.As(); @@ -754,7 +764,7 @@ public class PythonDerivedType } public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, string origMethodName, - object?[] args, RuntimeMethodHandle methodHandle) + object?[] args, RuntimeMethodHandle methodHandle, RuntimeTypeHandle declaringTypeHandle) { var self = GetPyObj(obj); if (null != self.Ref) @@ -779,7 +789,10 @@ public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, s } PyObject py_result = method.Invoke(pyargs); - MarshalByRefsBack(args, methodHandle, py_result, outsOffset: 0); + var clrMethod = methodHandle != default + ? MethodBase.GetMethodFromHandle(methodHandle, declaringTypeHandle) + : null; + MarshalByRefsBack(args, clrMethod, py_result, outsOffset: 0); return; } } @@ -811,12 +824,11 @@ public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, s /// as a tuple of new values for those arguments, and updates corresponding /// elements of array. /// - private static PyTuple? MarshalByRefsBack(object?[] args, RuntimeMethodHandle methodHandle, PyObject pyResult, int outsOffset) + private static PyTuple? MarshalByRefsBack(object?[] args, MethodBase? method, PyObject pyResult, int outsOffset) { - if (methodHandle == default) return null; + if (method is null) return null; - var originalMethod = MethodBase.GetMethodFromHandle(methodHandle); - var parameters = originalMethod.GetParameters(); + var parameters = method.GetParameters(); PyTuple? outs = null; int byrefIndex = 0; for (int i = 0; i < parameters.Length; ++i) diff --git a/src/runtime/Types/ManagedType.cs b/src/runtime/Types/ManagedType.cs index 2ed9d7970..97a19497c 100644 --- a/src/runtime/Types/ManagedType.cs +++ b/src/runtime/Types/ManagedType.cs @@ -148,8 +148,9 @@ protected static void ClearObjectDict(BorrowedReference ob) { BorrowedReference type = Runtime.PyObject_TYPE(ob); int instanceDictOffset = Util.ReadInt32(type, TypeOffset.tp_dictoffset); - Debug.Assert(instanceDictOffset > 0); - Runtime.Py_CLEAR(ob, instanceDictOffset); + // Debug.Assert(instanceDictOffset > 0); + if (instanceDictOffset > 0) + Runtime.Py_CLEAR(ob, instanceDictOffset); } protected static BorrowedReference GetObjectDict(BorrowedReference ob) diff --git a/src/runtime/Types/ReflectedClrType.cs b/src/runtime/Types/ReflectedClrType.cs index b787939be..d3d89bdb8 100644 --- a/src/runtime/Types/ReflectedClrType.cs +++ b/src/runtime/Types/ReflectedClrType.cs @@ -116,6 +116,6 @@ static ReflectedClrType AllocateClass(Type clrType) return new ReflectedClrType(type.Steal()); } - public override bool Equals(PyObject? other) => other != null && rawPtr == other.rawPtr; + public override bool Equals(PyObject? other) => rawPtr == other?.DangerousGetAddressOrNull(); public override int GetHashCode() => rawPtr.GetHashCode(); } diff --git a/src/runtime/Util/CodeGenerator.cs b/src/runtime/Util/CodeGenerator.cs index 35a637113..6e0859da0 100644 --- a/src/runtime/Util/CodeGenerator.cs +++ b/src/runtime/Util/CodeGenerator.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Globalization; +using System.Linq; using System.Reflection; using System.Reflection.Emit; using System.Threading; @@ -17,13 +19,15 @@ internal class CodeGenerator private readonly AssemblyBuilder aBuilder; private readonly ModuleBuilder mBuilder; + const string NamePrefix = "__Python_Runtime_Generated_"; + internal CodeGenerator() { - var aname = new AssemblyName { Name = "__CodeGenerator_Assembly" }; + var aname = new AssemblyName { Name = GetUniqueAssemblyName(NamePrefix + "Assembly") }; var aa = AssemblyBuilderAccess.Run; aBuilder = Thread.GetDomain().DefineDynamicAssembly(aname, aa); - mBuilder = aBuilder.DefineDynamicModule("__CodeGenerator_Module"); + mBuilder = aBuilder.DefineDynamicModule(NamePrefix + "Module"); } /// @@ -77,5 +81,20 @@ internal static void GenerateMarshalByRefsBack(ILGenerator il, IReadOnlyList(AppDomain.CurrentDomain + .GetAssemblies() + .Select(a => a.GetName().Name)); + for (int i = 0; i < int.MaxValue; i++) + { + string candidate = name + i.ToString(CultureInfo.InvariantCulture); + if (!taken.Contains(candidate)) + return candidate; + } + + throw new NotSupportedException("Too many assemblies"); + } } } diff --git a/src/runtime/Util/EventHandlerCollection.cs b/src/runtime/Util/EventHandlerCollection.cs index 551893799..0cd03d0fd 100644 --- a/src/runtime/Util/EventHandlerCollection.cs +++ b/src/runtime/Util/EventHandlerCollection.cs @@ -99,6 +99,10 @@ internal bool RemoveEventHandler(BorrowedReference target, BorrowedReference han continue; } list.RemoveAt(i); + if (list.Count == 0) + { + Remove(key); + } return true; } diff --git a/src/runtime/Util/PythonReferenceComparer.cs b/src/runtime/Util/PythonReferenceComparer.cs index dd78f912d..63c35df57 100644 --- a/src/runtime/Util/PythonReferenceComparer.cs +++ b/src/runtime/Util/PythonReferenceComparer.cs @@ -13,10 +13,10 @@ public sealed class PythonReferenceComparer : IEqualityComparer public static PythonReferenceComparer Instance { get; } = new PythonReferenceComparer(); public bool Equals(PyObject? x, PyObject? y) { - return x?.rawPtr == y?.rawPtr; + return x?.DangerousGetAddressOrNull() == y?.DangerousGetAddressOrNull(); } - public int GetHashCode(PyObject obj) => obj.rawPtr.GetHashCode(); + public int GetHashCode(PyObject obj) => obj.DangerousGetAddressOrNull().GetHashCode(); private PythonReferenceComparer() { } } diff --git a/src/testing/interfacetest.cs b/src/testing/interfacetest.cs index 7c5d937b9..a1a7106b5 100644 --- a/src/testing/interfacetest.cs +++ b/src/testing/interfacetest.cs @@ -79,7 +79,7 @@ private interface IPrivate { } } - + public interface IOutArg { string MyMethod_Out(string name, out int index); @@ -93,4 +93,25 @@ public static int CallMyMethod_Out(IOutArg myInterface) return index; } } + + public interface IGenericInterface + { + public T Get(T x); + } + + public class SpecificInterfaceUser + { + public SpecificInterfaceUser(IGenericInterface some, int x) + { + some.Get(x); + } + } + + public class GenericInterfaceUser + { + public GenericInterfaceUser(IGenericInterface some, T x) + { + some.Get(x); + } + } } diff --git a/tests/conftest.py b/tests/conftest.py index e61e3680e..1ac20e1dd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -79,9 +79,10 @@ def pytest_configure(config): ["dotnet", "publish", "-f", fw, "-o", str(bin_path), str(test_proj_path)] ) - from pythonnet import load - - load(runtime_opt, **runtime_params) + import os + os.environ["PYTHONNET_RUNTIME"] = runtime_opt + for k, v in runtime_params.items(): + os.environ[f"PYTHONNET_{runtime_opt.upper()}_{k.upper()}"] = v import clr diff --git a/tests/test_import.py b/tests/test_import.py index 25877be15..877eacd84 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -15,7 +15,7 @@ def test_relative_missing_import(): def test_import_all_on_second_time(): """Test import all attributes after a normal import without '*'. - Due to import * only allowed at module level, the test body splited + Due to import * only allowed at module level, the test body splitted to a module file.""" from . import importtest del sys.modules[importtest.__name__] diff --git a/tests/test_subclass.py b/tests/test_subclass.py index fa82c3663..a51e89da3 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -9,7 +9,7 @@ import System import pytest from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, - FunctionsTest) + FunctionsTest, IGenericInterface) from System.Collections.Generic import List @@ -29,6 +29,17 @@ def bar(self, x, i): return InterfaceTestClass +def interface_generic_class_fixture(subnamespace): + + class GenericInterfaceImpl(IGenericInterface[int]): + __namespace__ = "Python.Test." + subnamespace + + def Get(self, x): + return x + + return GenericInterfaceImpl + + def derived_class_fixture(subnamespace): """Delay creation of class until test starts.""" @@ -306,3 +317,13 @@ class Derived(BaseClass): import gc gc.collect() + +def test_generic_interface(): + from System import Int32 + from Python.Test import GenericInterfaceUser, SpecificInterfaceUser + + GenericInterfaceImpl = interface_generic_class_fixture(test_generic_interface.__name__) + + obj = GenericInterfaceImpl() + SpecificInterfaceUser(obj, Int32(0)) + GenericInterfaceUser[Int32](obj, Int32(0)) diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py old mode 100644 new mode 100755 index 0c80c1904..78e4d45c2 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -13,40 +13,26 @@ - clang """ -from __future__ import print_function - -import logging import os +import shutil import sys import sysconfig import subprocess -if sys.version_info.major > 2: - from io import StringIO -else: - from StringIO import StringIO - +from io import StringIO +from pathlib import Path from pycparser import c_ast, c_parser -_log = logging.getLogger() -logging.basicConfig(level=logging.DEBUG) - -PY_MAJOR = sys.version_info[0] -PY_MINOR = sys.version_info[1] - # rename some members from their C name when generating the C# _typeoffset_member_renames = { "ht_name": "name", - "ht_qualname": "qualname" + "ht_qualname": "qualname", + "getitem": "spec_cache_getitem", } def _check_output(*args, **kwargs): - """Check output wrapper for py2/py3 compatibility""" - output = subprocess.check_output(*args, **kwargs) - if PY_MAJOR == 2: - return output - return output.decode("ascii") + return subprocess.check_output(*args, **kwargs, encoding="utf8") class AstParser(object): @@ -92,7 +78,7 @@ def visit(self, node): self.visit_identifier(node) def visit_ast(self, ast): - for name, node in ast.children(): + for _name, node in ast.children(): self.visit(node) def visit_typedef(self, typedef): @@ -113,7 +99,7 @@ def visit_struct(self, struct): self.visit(decl) self._struct_members_stack.pop(0) self._struct_stack.pop(0) - elif self._ptr_decl_depth: + elif self._ptr_decl_depth or self._struct_members_stack: # the struct is empty, but add it as a member to the current # struct as the current member maybe a pointer to it. self._add_struct_member(struct.name) @@ -141,7 +127,8 @@ def _add_struct_member(self, type_name): current_struct = self._struct_stack[0] member_name = self._struct_members_stack[0] struct_members = self._struct_members.setdefault( - self._get_struct_name(current_struct), []) + self._get_struct_name(current_struct), [] + ) # get the node associated with this type node = None @@ -179,7 +166,6 @@ def _get_struct_name(self, node): class Writer(object): - def __init__(self): self._stream = StringIO() @@ -193,34 +179,47 @@ def to_string(self): return self._stream.getvalue() -def preprocess_python_headers(): +def preprocess_python_headers(*, cc=None, include_py=None): """Return Python.h pre-processed, ready for parsing. Requires clang. """ - fake_libc_include = os.path.join(os.path.dirname(__file__), - "fake_libc_include") + this_path = Path(__file__).parent + + fake_libc_include = this_path / "fake_libc_include" include_dirs = [fake_libc_include] - include_py = sysconfig.get_config_var("INCLUDEPY") + if cc is None: + cc = shutil.which("clang") + if cc is None: + cc = shutil.which("gcc") + if cc is None: + raise RuntimeError("No suitable C compiler found, need clang or gcc") + + if include_py is None: + include_py = sysconfig.get_config_var("INCLUDEPY") + include_py = Path(include_py) + include_dirs.append(include_py) - include_args = [c for p in include_dirs for c in ["-I", p]] + include_args = [c for p in include_dirs for c in ["-I", str(p)]] + # fmt: off defines = [ "-D", "__attribute__(x)=", "-D", "__inline__=inline", "-D", "__asm__=;#pragma asm", "-D", "__int64=long long", - "-D", "_POSIX_THREADS" + "-D", "_POSIX_THREADS", ] - if os.name == 'nt': + if sys.platform == "win32": defines.extend([ "-D", "__inline=inline", "-D", "__ptr32=", "-D", "__ptr64=", "-D", "__declspec(x)=", ]) + #fmt: on if hasattr(sys, "abiflags"): if "d" in sys.abiflags: @@ -228,8 +227,8 @@ def preprocess_python_headers(): if "u" in sys.abiflags: defines.extend(("-D", "PYTHON_WITH_WIDE_UNICODE")) - python_h = os.path.join(include_py, "Python.h") - cmd = ["clang", "-pthread"] + include_args + defines + ["-E", python_h] + python_h = include_py / "Python.h" + cmd = [cc, "-pthread"] + include_args + defines + ["-E", str(python_h)] # normalize as the parser doesn't like windows line endings. lines = [] @@ -240,16 +239,13 @@ def preprocess_python_headers(): return "\n".join(lines) - -def gen_interop_head(writer): +def gen_interop_head(writer, version, abi_flags): filename = os.path.basename(__file__) - abi_flags = getattr(sys, "abiflags", "").replace("m", "") - py_ver = "{0}.{1}".format(PY_MAJOR, PY_MINOR) - class_definition = """ -// Auto-generated by %s. + class_definition = f""" +// Auto-generated by {filename}. // DO NOT MODIFY BY HAND. -// Python %s: ABI flags: '%s' +// Python {".".join(version[:2])}: ABI flags: '{abi_flags}' // ReSharper disable InconsistentNaming // ReSharper disable IdentifierTypo @@ -261,7 +257,7 @@ def gen_interop_head(writer): using Python.Runtime.Native; namespace Python.Runtime -{""" % (filename, py_ver, abi_flags) +{{""" writer.extend(class_definition) @@ -271,25 +267,24 @@ def gen_interop_tail(writer): writer.extend(tail) -def gen_heap_type_members(parser, writer, type_name = None): +def gen_heap_type_members(parser, writer, type_name): """Generate the TypeOffset C# class""" members = parser.get_struct_members("PyHeapTypeObject") - type_name = type_name or "TypeOffset{0}{1}".format(PY_MAJOR, PY_MINOR) - class_definition = """ + class_definition = f""" [SuppressMessage("Style", "IDE1006:Naming Styles", Justification = "Following CPython", Scope = "type")] [StructLayout(LayoutKind.Sequential)] - internal class {0} : GeneratedTypeOffsets, ITypeOffsets + internal class {type_name} : GeneratedTypeOffsets, ITypeOffsets {{ - public {0}() {{ }} + public {type_name}() {{ }} // Auto-generated from PyHeapTypeObject in Python.h -""".format(type_name) +""" # All the members are sizeof(void*) so we don't need to do any # extra work to determine the size based on the type. - for name, tpy in members: + for name, _type in members: name = _typeoffset_member_renames.get(name, name) class_definition += " public int %s { get; private set; }\n" % name @@ -304,17 +299,18 @@ def gen_structure_code(parser, writer, type_name, indent): return False out = writer.append out(indent, "[StructLayout(LayoutKind.Sequential)]") - out(indent, "internal struct %s" % type_name) + out(indent, f"internal struct {type_name}") out(indent, "{") - for name, tpy in members: - out(indent + 1, "public IntPtr %s;" % name) + for name, _type in members: + out(indent + 1, f"public IntPtr {name};") out(indent, "}") out() return True -def main(): + +def main(*, cc=None, include_py=None, version=None, out=None): # preprocess Python.h and build the AST - python_h = preprocess_python_headers() + python_h = preprocess_python_headers(cc=cc, include_py=include_py) parser = c_parser.CParser() ast = parser.parse(python_h) @@ -323,21 +319,47 @@ def main(): ast_parser.visit(ast) writer = Writer() + + if include_py and not version: + raise RuntimeError("If the include path is overridden, version must be " + "defined" + ) + + if version: + version = version.split('.') + else: + version = sys.version_info + # generate the C# code - offsets_type_name = "NativeTypeOffset" if len(sys.argv) > 1 else None - gen_interop_head(writer) + abi_flags = getattr(sys, "abiflags", "").replace("m", "") + gen_interop_head(writer, version, abi_flags) - gen_heap_type_members(ast_parser, writer, type_name = offsets_type_name) + type_name = f"TypeOffset{version[0]}{version[1]}{abi_flags}" + gen_heap_type_members(ast_parser, writer, type_name) gen_interop_tail(writer) interop_cs = writer.to_string() - if len(sys.argv) > 1: - with open(sys.argv[1], "w") as fh: - fh.write(interop_cs) - else: + if not out or out == "-": print(interop_cs) + else: + with open(out, "w") as fh: + fh.write(interop_cs) if __name__ == "__main__": - sys.exit(main()) + import argparse + + a = argparse.ArgumentParser("Interop file generator for Python.NET") + a.add_argument("--cc", help="C compiler to use, either clang or gcc") + a.add_argument("--include-py", help="Include path of Python") + a.add_argument("--version", help="Python version") + a.add_argument("--out", help="Output path", default="-") + args = a.parse_args() + + sys.exit(main( + cc=args.cc, + include_py=args.include_py, + out=args.out, + version=args.version + )) diff --git a/version.txt b/version.txt index 4a36342fc..cb2b00e4f 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.0.0 +3.0.1