From 21c5dcc950272d84cb739bc6d61ba4d280882180 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Thu, 30 Jun 2022 15:24:00 -0700 Subject: [PATCH 01/34] implemented dynamic equality and inequality for PyObject instances fixed unhandled Python errors during comparison attempts fixes https://github.com/pythonnet/pythonnet/issues/1848 --- CHANGELOG.md | 2 + src/embed_tests/dynamic.cs | 22 +++++++++ src/runtime/PythonTypes/PyObject.cs | 75 +++++++++++++++++++++++------ src/runtime/Runtime.cs | 25 ---------- 4 files changed, 83 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52ee08484..9b5dd1816 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,8 @@ details about the cause of the failure able to access members that are part of the implementation class, but not the interface. Use the new `__implementation__` or `__raw_implementation__` properties to if you need to "downcast" to the implementation class. +- BREAKING: `==` and `!=` operators on `PyObject` instances now use Python comparison + (previously was equivalent to `object.ReferenceEquals(,)`) - BREAKING: Parameters marked with `ParameterAttributes.Out` are no longer returned in addition to the regular method return value (unless they are passed with `ref` or `out` keyword). - BREAKING: Drop support for the long-deprecated CLR.* prefix. diff --git a/src/embed_tests/dynamic.cs b/src/embed_tests/dynamic.cs index 0a181231c..6e3bfc4cb 100644 --- a/src/embed_tests/dynamic.cs +++ b/src/embed_tests/dynamic.cs @@ -128,6 +128,28 @@ public void PassPyObjectInNet() Assert.IsTrue(sys.testattr1.Equals(sys.testattr2)); } + // regression test for https://github.com/pythonnet/pythonnet/issues/1848 + [Test] + public void EnumEquality() + { + using var scope = Py.CreateScope(); + scope.Exec(@" +import enum + +class MyEnum(enum.IntEnum): + OK = 1 + ERROR = 2 + +def get_status(): + return MyEnum.OK +" +); + + dynamic MyEnum = scope.Get("MyEnum"); + dynamic status = scope.Get("get_status").Invoke(); + Assert.IsTrue(status == MyEnum.OK); + } + // regression test for https://github.com/pythonnet/pythonnet/issues/1680 [Test] public void ForEach() diff --git a/src/runtime/PythonTypes/PyObject.cs b/src/runtime/PythonTypes/PyObject.cs index cfd3e7158..3d48e22ed 100644 --- a/src/runtime/PythonTypes/PyObject.cs +++ b/src/runtime/PythonTypes/PyObject.cs @@ -1075,12 +1075,9 @@ public virtual bool Equals(PyObject? other) { return true; } - int r = Runtime.PyObject_Compare(this, other); - if (Exceptions.ErrorOccurred()) - { - throw PythonException.ThrowLastAsClrException(); - } - return r == 0; + int result = Runtime.PyObject_RichCompareBool(obj, other.obj, Runtime.Py_EQ); + if (result < 0) throw PythonException.ThrowLastAsClrException(); + return result != 0; } @@ -1304,6 +1301,18 @@ public override bool TryConvert(ConvertBinder binder, out object? result) return false; } + private bool TryCompare(PyObject arg, int op, out object @out) + { + int result = Runtime.PyObject_RichCompareBool(this.obj, arg.obj, op); + @out = result != 0; + if (result < 0) + { + Exceptions.Clear(); + return false; + } + return true; + } + public override bool TryBinaryOperation(BinaryOperationBinder binder, object arg, out object? result) { using var _ = Py.GIL(); @@ -1352,11 +1361,9 @@ public override bool TryBinaryOperation(BinaryOperationBinder binder, object arg res = Runtime.PyNumber_InPlaceXor(this.obj, ((PyObject)arg).obj); break; case ExpressionType.GreaterThan: - result = Runtime.PyObject_Compare(this.obj, ((PyObject)arg).obj) > 0; - return true; + return this.TryCompare((PyObject)arg, Runtime.Py_GT, out result); case ExpressionType.GreaterThanOrEqual: - result = Runtime.PyObject_Compare(this.obj, ((PyObject)arg).obj) >= 0; - return true; + return this.TryCompare((PyObject)arg, Runtime.Py_GE, out result); case ExpressionType.LeftShift: res = Runtime.PyNumber_Lshift(this.obj, ((PyObject)arg).obj); break; @@ -1364,11 +1371,9 @@ public override bool TryBinaryOperation(BinaryOperationBinder binder, object arg res = Runtime.PyNumber_InPlaceLshift(this.obj, ((PyObject)arg).obj); break; case ExpressionType.LessThan: - result = Runtime.PyObject_Compare(this.obj, ((PyObject)arg).obj) < 0; - return true; + return this.TryCompare((PyObject)arg, Runtime.Py_LT, out result); case ExpressionType.LessThanOrEqual: - result = Runtime.PyObject_Compare(this.obj, ((PyObject)arg).obj) <= 0; - return true; + return this.TryCompare((PyObject)arg, Runtime.Py_LE, out result); case ExpressionType.Modulo: res = Runtime.PyNumber_Remainder(this.obj, ((PyObject)arg).obj); break; @@ -1376,8 +1381,9 @@ public override bool TryBinaryOperation(BinaryOperationBinder binder, object arg res = Runtime.PyNumber_InPlaceRemainder(this.obj, ((PyObject)arg).obj); break; case ExpressionType.NotEqual: - result = Runtime.PyObject_Compare(this.obj, ((PyObject)arg).obj) != 0; - return true; + return this.TryCompare((PyObject)arg, Runtime.Py_NE, out result); + case ExpressionType.Equal: + return this.TryCompare((PyObject)arg, Runtime.Py_EQ, out result); case ExpressionType.Or: res = Runtime.PyNumber_Or(this.obj, ((PyObject)arg).obj); break; @@ -1402,6 +1408,40 @@ public override bool TryBinaryOperation(BinaryOperationBinder binder, object arg return true; } + public static bool operator ==(PyObject? a, PyObject? b) + { + if (a is null && b is null) + { + return true; + } + if (a is null || b is null) + { + return false; + } + + using var _ = Py.GIL(); + int result = Runtime.PyObject_RichCompareBool(a.obj, b.obj, Runtime.Py_EQ); + if (result < 0) throw PythonException.ThrowLastAsClrException(); + return result != 0; + } + + public static bool operator !=(PyObject? a, PyObject? b) + { + if (a is null && b is null) + { + return false; + } + if (a is null || b is null) + { + return true; + } + + using var _ = Py.GIL(); + int result = Runtime.PyObject_RichCompareBool(a.obj, b.obj, Runtime.Py_NE); + if (result < 0) throw PythonException.ThrowLastAsClrException(); + return result != 0; + } + // Workaround for https://bugzilla.xamarin.com/show_bug.cgi?id=41509 // See https://github.com/pythonnet/pythonnet/pull/219 internal static object? CheckNone(PyObject pyObj) @@ -1436,14 +1476,17 @@ public override bool TryUnaryOperation(UnaryOperationBinder binder, out object? case ExpressionType.Not: r = Runtime.PyObject_Not(this.obj); result = r == 1; + if (r == -1) Exceptions.Clear(); return r != -1; case ExpressionType.IsFalse: r = Runtime.PyObject_IsTrue(this.obj); result = r == 0; + if (r == -1) Exceptions.Clear(); return r != -1; case ExpressionType.IsTrue: r = Runtime.PyObject_IsTrue(this.obj); result = r == 1; + if (r == -1) Exceptions.Clear(); return r != -1; case ExpressionType.Decrement: case ExpressionType.Increment: diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 6ad1d459f..1eeb96b54 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -962,31 +962,6 @@ internal static IntPtr PyObject_CallObject(IntPtr pointer, IntPtr args) internal static int PyObject_RichCompareBool(BorrowedReference value1, BorrowedReference value2, int opid) => Delegates.PyObject_RichCompareBool(value1, value2, opid); - internal static int PyObject_Compare(BorrowedReference value1, BorrowedReference value2) - { - int res; - res = PyObject_RichCompareBool(value1, value2, Py_LT); - if (-1 == res) - return -1; - else if (1 == res) - return -1; - - res = PyObject_RichCompareBool(value1, value2, Py_EQ); - if (-1 == res) - return -1; - else if (1 == res) - return 0; - - res = PyObject_RichCompareBool(value1, value2, Py_GT); - if (-1 == res) - return -1; - else if (1 == res) - return 1; - - Exceptions.SetError(Exceptions.SystemError, "Error comparing objects"); - return -1; - } - internal static int PyObject_IsInstance(BorrowedReference ob, BorrowedReference type) => Delegates.PyObject_IsInstance(ob, type); From e241a9d1ccd6b5168aed2457a6c96117d708306f Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Thu, 30 Jun 2022 16:25:15 -0700 Subject: [PATCH 02/34] got rid of a few deprecation warnings that pollute GitHub code review --- src/embed_tests/TestConverter.cs | 9 ++++++--- src/embed_tests/TestDomainReload.cs | 3 +-- src/embed_tests/TestFinalizer.cs | 7 +++++-- src/embed_tests/TestNativeTypeOffset.cs | 3 ++- src/embed_tests/TestPythonException.cs | 2 +- src/runtime/InternString.cs | 4 ++-- src/runtime/PythonTypes/PyObject.cs | 4 +++- src/runtime/Types/ReflectedClrType.cs | 2 +- src/runtime/Util/PythonReferenceComparer.cs | 4 ++-- 9 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/embed_tests/TestConverter.cs b/src/embed_tests/TestConverter.cs index e586eda1b..0686d528b 100644 --- a/src/embed_tests/TestConverter.cs +++ b/src/embed_tests/TestConverter.cs @@ -148,7 +148,7 @@ public void PyIntImplicit() { var i = new PyInt(1); var ni = (PyObject)i.As(); - Assert.AreEqual(i.rawPtr, ni.rawPtr); + Assert.IsTrue(PythonReferenceComparer.Instance.Equals(i, ni)); } [Test] @@ -178,8 +178,11 @@ public void RawPyObjectProxy() var clrObject = (CLRObject)ManagedType.GetManagedObject(pyObjectProxy); Assert.AreSame(pyObject, clrObject.inst); - var proxiedHandle = pyObjectProxy.GetAttr("Handle").As(); - Assert.AreEqual(pyObject.Handle, proxiedHandle); +#pragma warning disable CS0612 // Type or member is obsolete + const string handlePropertyName = nameof(PyObject.Handle); +#pragma warning restore CS0612 // Type or member is obsolete + var proxiedHandle = pyObjectProxy.GetAttr(handlePropertyName).As(); + Assert.AreEqual(pyObject.DangerousGetAddressOrNull(), proxiedHandle); } // regression for https://github.com/pythonnet/pythonnet/issues/451 diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs index 498119d1e..a0f9b63eb 100644 --- a/src/embed_tests/TestDomainReload.cs +++ b/src/embed_tests/TestDomainReload.cs @@ -99,8 +99,7 @@ from Python.EmbeddingTest.Domain import MyClass { Debug.Assert(obj.AsManagedObject(type).GetType() == type); // We only needs its Python handle - PyRuntime.XIncref(obj); - return obj.Handle; + return new NewReference(obj).DangerousMoveToPointer(); } } } diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 40ab03395..b748a2244 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -212,7 +212,9 @@ public void ValidateRefCount() Assert.AreEqual(ptr, e.Handle); Assert.AreEqual(2, e.ImpactedObjects.Count); // Fix for this test, don't do this on general environment +#pragma warning disable CS0618 // Type or member is obsolete Runtime.Runtime.XIncref(e.Reference); +#pragma warning restore CS0618 // Type or member is obsolete return false; }; Finalizer.Instance.IncorrectRefCntResolver += handler; @@ -234,8 +236,9 @@ private static IntPtr CreateStringGarbage() { PyString s1 = new PyString("test_string"); // s2 steal a reference from s1 - PyString s2 = new PyString(StolenReference.DangerousFromPointer(s1.Handle)); - return s1.Handle; + IntPtr address = s1.Reference.DangerousGetAddress(); + PyString s2 = new (StolenReference.DangerousFromPointer(address)); + return address; } } } diff --git a/src/embed_tests/TestNativeTypeOffset.cs b/src/embed_tests/TestNativeTypeOffset.cs index 2d31fe506..d692c24e6 100644 --- a/src/embed_tests/TestNativeTypeOffset.cs +++ b/src/embed_tests/TestNativeTypeOffset.cs @@ -33,7 +33,8 @@ public void LoadNativeTypeOffsetClass() { PyObject sys = Py.Import("sys"); // We can safely ignore the "m" abi flag - var abiflags = sys.GetAttr("abiflags", "".ToPython()).ToString().Replace("m", ""); + var abiflags = sys.HasAttr("abiflags") ? sys.GetAttr("abiflags").ToString() : ""; + abiflags = abiflags.Replace("m", ""); if (!string.IsNullOrEmpty(abiflags)) { string typeName = "Python.Runtime.NativeTypeOffset, Python.Runtime"; diff --git a/src/embed_tests/TestPythonException.cs b/src/embed_tests/TestPythonException.cs index a7cf05c83..a248b6a1f 100644 --- a/src/embed_tests/TestPythonException.cs +++ b/src/embed_tests/TestPythonException.cs @@ -161,7 +161,7 @@ def __init__(self, val): using var tbObj = tbPtr.MoveToPyObject(); // the type returned from PyErr_NormalizeException should not be the same type since a new // exception was raised by initializing the exception - Assert.AreNotEqual(type.Handle, typeObj.Handle); + Assert.IsFalse(PythonReferenceComparer.Instance.Equals(type, typeObj)); // the message should now be the string from the throw exception during normalization Assert.AreEqual("invalid literal for int() with base 10: 'dummy string'", strObj.ToString()); } diff --git a/src/runtime/InternString.cs b/src/runtime/InternString.cs index b6d9a0e4a..decb3981d 100644 --- a/src/runtime/InternString.cs +++ b/src/runtime/InternString.cs @@ -42,7 +42,7 @@ public static void Initialize() Debug.Assert(name == op.As()); SetIntern(name, op); var field = type.GetField("f" + name, PyIdentifierFieldFlags)!; - field.SetValue(null, op.rawPtr); + field.SetValue(null, op.DangerousGetAddressOrNull()); } } @@ -76,7 +76,7 @@ public static bool TryGetInterned(BorrowedReference op, out string s) private static void SetIntern(string s, PyString op) { _string2interns.Add(s, op); - _intern2strings.Add(op.rawPtr, s); + _intern2strings.Add(op.Reference.DangerousGetAddress(), s); } } } diff --git a/src/runtime/PythonTypes/PyObject.cs b/src/runtime/PythonTypes/PyObject.cs index 3d48e22ed..ce86753eb 100644 --- a/src/runtime/PythonTypes/PyObject.cs +++ b/src/runtime/PythonTypes/PyObject.cs @@ -27,7 +27,7 @@ public partial class PyObject : DynamicObject, IDisposable, ISerializable public StackTrace Traceback { get; } = new StackTrace(1); #endif - protected internal IntPtr rawPtr = IntPtr.Zero; + protected IntPtr rawPtr = IntPtr.Zero; internal readonly int run = Runtime.GetRun(); internal BorrowedReference obj => new (rawPtr); @@ -252,6 +252,8 @@ internal void Leak() rawPtr = IntPtr.Zero; } + internal IntPtr DangerousGetAddressOrNull() => rawPtr; + internal void CheckRun() { if (run != Runtime.GetRun()) diff --git a/src/runtime/Types/ReflectedClrType.cs b/src/runtime/Types/ReflectedClrType.cs index 2e8f95924..e92a28018 100644 --- a/src/runtime/Types/ReflectedClrType.cs +++ b/src/runtime/Types/ReflectedClrType.cs @@ -117,6 +117,6 @@ static ReflectedClrType AllocateClass(Type clrType) return new ReflectedClrType(type.Steal()); } - public override bool Equals(PyObject? other) => other != null && rawPtr == other.rawPtr; + public override bool Equals(PyObject? other) => rawPtr == other?.DangerousGetAddressOrNull(); public override int GetHashCode() => rawPtr.GetHashCode(); } diff --git a/src/runtime/Util/PythonReferenceComparer.cs b/src/runtime/Util/PythonReferenceComparer.cs index dd78f912d..63c35df57 100644 --- a/src/runtime/Util/PythonReferenceComparer.cs +++ b/src/runtime/Util/PythonReferenceComparer.cs @@ -13,10 +13,10 @@ public sealed class PythonReferenceComparer : IEqualityComparer public static PythonReferenceComparer Instance { get; } = new PythonReferenceComparer(); public bool Equals(PyObject? x, PyObject? y) { - return x?.rawPtr == y?.rawPtr; + return x?.DangerousGetAddressOrNull() == y?.DangerousGetAddressOrNull(); } - public int GetHashCode(PyObject obj) => obj.rawPtr.GetHashCode(); + public int GetHashCode(PyObject obj) => obj.DangerousGetAddressOrNull().GetHashCode(); private PythonReferenceComparer() { } } From ce3afa64c85e5bf88a54d581eae4719810f85e19 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Tue, 28 Jun 2022 14:45:43 +0200 Subject: [PATCH 03/34] Fix broken prefix and debug leftover Additionally, fixes a type hint and makes sure that the new default behaviour is to use the environment variable if given. --- pythonnet/__init__.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pythonnet/__init__.py b/pythonnet/__init__.py index fa6ed45cf..9876a0bec 100644 --- a/pythonnet/__init__.py +++ b/pythonnet/__init__.py @@ -3,7 +3,7 @@ from typing import Dict, Optional, Union import clr_loader -__all__ = ["set_runtime", "set_default_runtime", "load"] +__all__ = ["set_runtime", "set_runtime_from_env", "load"] _RUNTIME: Optional[clr_loader.Runtime] = None _LOADER_ASSEMBLY: Optional[clr_loader.wrappers.Assembly] = None @@ -30,7 +30,7 @@ def set_runtime(runtime: Union[clr_loader.Runtime, str], **params: str) -> None: def _get_params_from_env(prefix: str) -> Dict[str, str]: from os import environ - full_prefix = f"PYTHONNET_{prefix.upper()}" + full_prefix = f"PYTHONNET_{prefix.upper()}_" len_ = len(full_prefix) env_vars = { @@ -63,8 +63,8 @@ def _create_runtime_from_spec( raise RuntimeError(f"Invalid runtime name: '{spec}'") -def set_default_runtime() -> None: - """Set up the default runtime +def set_runtime_from_env() -> None: + """Set up the runtime using the environment This will use the environment variable PYTHONNET_RUNTIME to decide the runtime to use, which may be one of netfx, coreclr or mono. The parameters @@ -80,16 +80,13 @@ def set_default_runtime() -> None: """ from os import environ - print("Set default RUNTIME") - raise RuntimeError("Shouldn't be called here") - spec = environ.get("PYTHONNET_RUNTIME", "default") runtime = _create_runtime_from_spec(spec) set_runtime(runtime) def load( - runtime: Union[clr_loader.Runtime, str] = "default", **params: Dict[str, str] + runtime: Union[clr_loader.Runtime, str, None] = None, **params: str ) -> None: """Load Python.NET in the specified runtime @@ -102,7 +99,10 @@ def load( return if _RUNTIME is None: - set_runtime(runtime, **params) + if runtime is None: + set_runtime_from_env() + else: + set_runtime(runtime, **params) if _RUNTIME is None: raise RuntimeError("No valid runtime selected") From 14aae2ebc8de4fd5f9eaac520573d89bbfc89a3e Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 6 Jul 2022 08:32:51 +0200 Subject: [PATCH 04/34] Ensure that version.txt is always read from repo root Allows the project to be referenced in other .NET projects without adjusting its project file (#1853). --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index 8c5b53685..965610f91 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -6,7 +6,7 @@ Python.NET 10.0 false - $([System.IO.File]::ReadAllText("version.txt")) + $([System.IO.File]::ReadAllText($(MSBuildThisFileDirectory)version.txt)) $(FullVersion.Split('-', 2)[0]) $(FullVersion.Split('-', 2)[1]) From 60463a31b28b645114e17f9f55a8283d9ca74257 Mon Sep 17 00:00:00 2001 From: Tim Gates Date: Sun, 17 Jul 2022 09:09:05 +1000 Subject: [PATCH 05/34] docs: Fix a few typos There are small typos in: - pythonnet/__init__.py - tests/test_import.py Fixes: - Should read `splitted` rather than `splited`. - Should read `loaded` rather than `laoded`. Signed-off-by: Tim Gates --- pythonnet/__init__.py | 2 +- tests/test_import.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pythonnet/__init__.py b/pythonnet/__init__.py index 9876a0bec..8f3478713 100644 --- a/pythonnet/__init__.py +++ b/pythonnet/__init__.py @@ -121,7 +121,7 @@ def load( def unload() -> None: - """Explicitly unload a laoded runtime and shut down Python.NET""" + """Explicitly unload a loaded runtime and shut down Python.NET""" global _RUNTIME, _LOADER_ASSEMBLY if _LOADER_ASSEMBLY is not None: diff --git a/tests/test_import.py b/tests/test_import.py index 25877be15..877eacd84 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -15,7 +15,7 @@ def test_relative_missing_import(): def test_import_all_on_second_time(): """Test import all attributes after a normal import without '*'. - Due to import * only allowed at module level, the test body splited + Due to import * only allowed at module level, the test body splitted to a module file.""" from . import importtest del sys.modules[importtest.__name__] From 2c233efbeb1c78efb80e0b8066c5d2b5ce2eed0e Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 29 Sep 2022 17:13:43 +0200 Subject: [PATCH 06/34] Bump master version to 3.1.0-dev --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 4a36342fc..0f9d6b15d 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.0.0 +3.1.0-dev From eb62787ff39792cb4b37b02a399bc8fe0e5b4a18 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 29 Sep 2022 17:14:14 +0200 Subject: [PATCH 07/34] Run documentation build from master --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 21402a42e..5b782c8b4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -24,7 +24,7 @@ jobs: path: doc/build/html/ deploy: - if: github.ref == 'refs/heads/release' + if: github.ref == 'refs/heads/master' runs-on: ubuntu-latest permissions: contents: read From a3b7789b6a575913941b74fe101e6529b0b70b9f Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 29 Sep 2022 17:20:49 +0200 Subject: [PATCH 08/34] Roll changelog over --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f0f212e6..caadec60d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added +### Changed + +### Fixed + + +## [3.0.0][] - 2022-09-29 + +### Added + - Ability to instantiate new .NET arrays using `Array[T](dim1, dim2, ...)` syntax - Python operator method will call C# operator method for supported binary and unary operators ([#1324][p1324]). - Add GetPythonThreadID and Interrupt methods in PythonEngine From 0119f1caa60462b89754f4ac052fa1b2e0e32cc3 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Fri, 30 Sep 2022 00:06:52 +0200 Subject: [PATCH 09/34] Specify supported Python version in project metadata --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 39c7c14fb..91f386fc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,8 @@ dependencies = [ "clr_loader>=0.2.2,<0.3.0" ] +requires-python = ">=3.7, <3.11" + classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", From bf984f09b36f74b564528de82e5a33d6a6389a00 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Fri, 30 Sep 2022 02:09:28 +0200 Subject: [PATCH 10/34] Merge backports-2.5 changelog in and fix links --- CHANGELOG.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index caadec60d..5d33ee327 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Fixed -## [3.0.0][] - 2022-09-29 +## [3.0.0](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.0) - 2022-09-29 ### Added @@ -143,7 +143,24 @@ There is no need to specify it. - support for .NET Framework 4.0-4.6; Mono before 5.4. Python.NET now requires .NET Standard 2.0 (see [the matrix](https://docs.microsoft.com/en-us/dotnet/standard/net-standard#net-implementation-support)) -## [2.5.0][] - 2020-06-14 +## [2.5.2](https://github.com/pythonnet/pythonnet/releases/tag/v2.5.2) - 2021-02-05 + +Bugfix release. + +### Fixed +- Fix `object[]` parameters taking precedence when should not in overload resolution +- Empty parameter names (as can be generated from F#) do not cause crashes + +## [2.5.1](https://github.com/pythonnet/pythonnet/releases/tag/v2.5.1) - 2020-06-18 + +Bugfix release. + +### Fixed + +- Fix incorrect dereference of wrapper object in `tp_repr`, which may result in a program crash +- Fix incorrect dereference in params array handling + +## [2.5.0](https://github.com/pythonnet/pythonnet/releases/tag/v2.5.0) - 2020-06-14 This version improves performance on benchmarks significantly compared to 2.3. From b5c222ca7bda1e9d37c788286356a71db1f6e124 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Thu, 13 Oct 2022 12:06:26 -0700 Subject: [PATCH 11/34] a few debug helper properties --- src/runtime/Finalizer.cs | 2 ++ src/runtime/PythonTypes/PyObject.cs | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/runtime/Finalizer.cs b/src/runtime/Finalizer.cs index f4b465ecb..713564f08 100644 --- a/src/runtime/Finalizer.cs +++ b/src/runtime/Finalizer.cs @@ -364,6 +364,8 @@ struct PendingFinalization { public IntPtr PyObj; public BorrowedReference Ref => new(PyObj); + public ManagedType? Managed => ManagedType.GetManagedObject(Ref); + public nint RefCount => Runtime.Refcount(Ref); public int RuntimeRun; #if TRACE_ALLOC public string StackTrace; diff --git a/src/runtime/PythonTypes/PyObject.cs b/src/runtime/PythonTypes/PyObject.cs index ce86753eb..bda2d9c02 100644 --- a/src/runtime/PythonTypes/PyObject.cs +++ b/src/runtime/PythonTypes/PyObject.cs @@ -1051,9 +1051,20 @@ public PyList Dir() return Runtime.GetManagedString(strval.BorrowOrThrow()); } - string? DebuggerDisplay => DebugUtil.HaveInterpreterLock() - ? this.ToString() - : $"pyobj at 0x{this.rawPtr:X} (get Py.GIL to see more info)"; + ManagedType? InternalManagedObject => ManagedType.GetManagedObject(this.Reference); + + string? DebuggerDisplay + { + get + { + if (DebugUtil.HaveInterpreterLock()) + return this.ToString(); + var obj = this.InternalManagedObject; + return obj is { } + ? obj.ToString() + : $"pyobj at 0x{this.rawPtr:X} (get Py.GIL to see more info)"; + } + } /// From 6838ee1ce2866d04d4a516801b1f2f7bc1aa2d55 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Thu, 13 Oct 2022 13:36:17 -0700 Subject: [PATCH 12/34] fixed name collision for generated delegate dispatchers --- src/runtime/Util/CodeGenerator.cs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/runtime/Util/CodeGenerator.cs b/src/runtime/Util/CodeGenerator.cs index 35a637113..6e0859da0 100644 --- a/src/runtime/Util/CodeGenerator.cs +++ b/src/runtime/Util/CodeGenerator.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Globalization; +using System.Linq; using System.Reflection; using System.Reflection.Emit; using System.Threading; @@ -17,13 +19,15 @@ internal class CodeGenerator private readonly AssemblyBuilder aBuilder; private readonly ModuleBuilder mBuilder; + const string NamePrefix = "__Python_Runtime_Generated_"; + internal CodeGenerator() { - var aname = new AssemblyName { Name = "__CodeGenerator_Assembly" }; + var aname = new AssemblyName { Name = GetUniqueAssemblyName(NamePrefix + "Assembly") }; var aa = AssemblyBuilderAccess.Run; aBuilder = Thread.GetDomain().DefineDynamicAssembly(aname, aa); - mBuilder = aBuilder.DefineDynamicModule("__CodeGenerator_Module"); + mBuilder = aBuilder.DefineDynamicModule(NamePrefix + "Module"); } /// @@ -77,5 +81,20 @@ internal static void GenerateMarshalByRefsBack(ILGenerator il, IReadOnlyList(AppDomain.CurrentDomain + .GetAssemblies() + .Select(a => a.GetName().Name)); + for (int i = 0; i < int.MaxValue; i++) + { + string candidate = name + i.ToString(CultureInfo.InvariantCulture); + if (!taken.Contains(candidate)) + return candidate; + } + + throw new NotSupportedException("Too many assemblies"); + } } } From 5a28fd41db0bd1a78477717dbb91f1a10703a074 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Thu, 13 Oct 2022 12:07:39 -0700 Subject: [PATCH 13/34] delete target object from event handler collections when it has no more event handlers fixes https://github.com/pythonnet/pythonnet/issues/1972 --- CHANGELOG.md | 2 + src/embed_tests/Events.cs | 67 ++++++++++++++++++++++ src/runtime/Util/EventHandlerCollection.cs | 4 ++ 3 files changed, 73 insertions(+) create mode 100644 src/embed_tests/Events.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d33ee327..9781f289c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Fixed +- Fixed objects leaking when Python attached event handlers to them even if they were later removed + ## [3.0.0](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.0) - 2022-09-29 diff --git a/src/embed_tests/Events.cs b/src/embed_tests/Events.cs new file mode 100644 index 000000000..c216f4214 --- /dev/null +++ b/src/embed_tests/Events.cs @@ -0,0 +1,67 @@ +using System; +using System.Diagnostics; +using System.Threading; + +using NUnit.Framework; + +using Python.Runtime; + +namespace Python.EmbeddingTest; + +public class Events +{ + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void UsingDoesNotLeak() + { + using var scope = Py.CreateScope(); + scope.Exec(@" +import gc + +from Python.EmbeddingTest import ClassWithEventHandler + +def event_handler(): + pass + +for _ in range(2000): + example = ClassWithEventHandler() + example.LeakEvent += event_handler + example.LeakEvent -= event_handler + del example + +gc.collect() +"); + Runtime.Runtime.TryCollectingGarbage(10); + Assert.AreEqual(0, ClassWithEventHandler.alive); + } +} + +public class ClassWithEventHandler +{ + internal static int alive; + + public event EventHandler LeakEvent; + private Array arr; // dummy array to exacerbate memory leak + + public ClassWithEventHandler() + { + Interlocked.Increment(ref alive); + this.arr = new int[800]; + } + + ~ClassWithEventHandler() + { + Interlocked.Decrement(ref alive); + } +} diff --git a/src/runtime/Util/EventHandlerCollection.cs b/src/runtime/Util/EventHandlerCollection.cs index 551893799..0cd03d0fd 100644 --- a/src/runtime/Util/EventHandlerCollection.cs +++ b/src/runtime/Util/EventHandlerCollection.cs @@ -99,6 +99,10 @@ internal bool RemoveEventHandler(BorrowedReference target, BorrowedReference han continue; } list.RemoveAt(i); + if (list.Count == 0) + { + Remove(key); + } return true; } From c422abdba471a114f04519c08abc7bba71163afb Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 18 Sep 2022 16:00:02 +0200 Subject: [PATCH 14/34] Add Python 3.11 type offsets --- src/runtime/Native/TypeOffset311.cs | 141 ++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 src/runtime/Native/TypeOffset311.cs diff --git a/src/runtime/Native/TypeOffset311.cs b/src/runtime/Native/TypeOffset311.cs new file mode 100644 index 000000000..7691236c9 --- /dev/null +++ b/src/runtime/Native/TypeOffset311.cs @@ -0,0 +1,141 @@ + +// Auto-generated by geninterop.py. +// DO NOT MODIFY BY HAND. + +// Python 3.11: ABI flags: '' + +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + + [StructLayout(LayoutKind.Sequential)] + internal class TypeOffset311 : GeneratedTypeOffsets, ITypeOffsets + { + public TypeOffset311() { } + // Auto-generated from PyHeapTypeObject in Python.h + public int ob_refcnt { get; private set; } + public int ob_type { get; private set; } + public int ob_size { get; private set; } + public int tp_name { get; private set; } + public int tp_basicsize { get; private set; } + public int tp_itemsize { get; private set; } + public int tp_dealloc { get; private set; } + public int tp_vectorcall_offset { get; private set; } + public int tp_getattr { get; private set; } + public int tp_setattr { get; private set; } + public int tp_as_async { get; private set; } + public int tp_repr { get; private set; } + public int tp_as_number { get; private set; } + public int tp_as_sequence { get; private set; } + public int tp_as_mapping { get; private set; } + public int tp_hash { get; private set; } + public int tp_call { get; private set; } + public int tp_str { get; private set; } + public int tp_getattro { get; private set; } + public int tp_setattro { get; private set; } + public int tp_as_buffer { get; private set; } + public int tp_flags { get; private set; } + public int tp_doc { get; private set; } + public int tp_traverse { get; private set; } + public int tp_clear { get; private set; } + public int tp_richcompare { get; private set; } + public int tp_weaklistoffset { get; private set; } + public int tp_iter { get; private set; } + public int tp_iternext { get; private set; } + public int tp_methods { get; private set; } + public int tp_members { get; private set; } + public int tp_getset { get; private set; } + public int tp_base { get; private set; } + public int tp_dict { get; private set; } + public int tp_descr_get { get; private set; } + public int tp_descr_set { get; private set; } + public int tp_dictoffset { get; private set; } + public int tp_init { get; private set; } + public int tp_alloc { get; private set; } + public int tp_new { get; private set; } + public int tp_free { get; private set; } + public int tp_is_gc { get; private set; } + public int tp_bases { get; private set; } + public int tp_mro { get; private set; } + public int tp_cache { get; private set; } + public int tp_subclasses { get; private set; } + public int tp_weaklist { get; private set; } + public int tp_del { get; private set; } + public int tp_version_tag { get; private set; } + public int tp_finalize { get; private set; } + public int tp_vectorcall { get; private set; } + public int am_await { get; private set; } + public int am_aiter { get; private set; } + public int am_anext { get; private set; } + public int am_send { get; private set; } + public int nb_add { get; private set; } + public int nb_subtract { get; private set; } + public int nb_multiply { get; private set; } + public int nb_remainder { get; private set; } + public int nb_divmod { get; private set; } + public int nb_power { get; private set; } + public int nb_negative { get; private set; } + public int nb_positive { get; private set; } + public int nb_absolute { get; private set; } + public int nb_bool { get; private set; } + public int nb_invert { get; private set; } + public int nb_lshift { get; private set; } + public int nb_rshift { get; private set; } + public int nb_and { get; private set; } + public int nb_xor { get; private set; } + public int nb_or { get; private set; } + public int nb_int { get; private set; } + public int nb_reserved { get; private set; } + public int nb_float { get; private set; } + public int nb_inplace_add { get; private set; } + public int nb_inplace_subtract { get; private set; } + public int nb_inplace_multiply { get; private set; } + public int nb_inplace_remainder { get; private set; } + public int nb_inplace_power { get; private set; } + public int nb_inplace_lshift { get; private set; } + public int nb_inplace_rshift { get; private set; } + public int nb_inplace_and { get; private set; } + public int nb_inplace_xor { get; private set; } + public int nb_inplace_or { get; private set; } + public int nb_floor_divide { get; private set; } + public int nb_true_divide { get; private set; } + public int nb_inplace_floor_divide { get; private set; } + public int nb_inplace_true_divide { get; private set; } + public int nb_index { get; private set; } + public int nb_matrix_multiply { get; private set; } + public int nb_inplace_matrix_multiply { get; private set; } + public int mp_length { get; private set; } + public int mp_subscript { get; private set; } + public int mp_ass_subscript { get; private set; } + public int sq_length { get; private set; } + public int sq_concat { get; private set; } + public int sq_repeat { get; private set; } + public int sq_item { get; private set; } + public int was_sq_slice { get; private set; } + public int sq_ass_item { get; private set; } + public int was_sq_ass_slice { get; private set; } + public int sq_contains { get; private set; } + public int sq_inplace_concat { get; private set; } + public int sq_inplace_repeat { get; private set; } + public int bf_getbuffer { get; private set; } + public int bf_releasebuffer { get; private set; } + public int name { get; private set; } + public int ht_slots { get; private set; } + public int qualname { get; private set; } + public int ht_cached_keys { get; private set; } + public int ht_module { get; private set; } + public int _ht_tpname { get; private set; } + } +} + From d6c024c8fb891024cbe7045268a0ca74351c547b Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Mon, 24 Oct 2022 09:53:59 -0700 Subject: [PATCH 15/34] explicit functions for exact type checks --- src/runtime/Runtime.cs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 6238119ff..7110f3cb0 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -1094,8 +1094,13 @@ internal static nint PyBuffer_SizeFromFormat(string format) internal static bool PyInt_Check(BorrowedReference ob) => PyObject_TypeCheck(ob, PyLongType); + internal static bool PyInt_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyLongType); + internal static bool PyBool_Check(BorrowedReference ob) => PyObject_TypeCheck(ob, PyBoolType); + internal static bool PyBool_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyBoolType); internal static NewReference PyInt_FromInt32(int value) => PyLong_FromLongLong(value); @@ -1141,6 +1146,8 @@ internal static NewReference PyLong_FromString(string value, int radix) internal static bool PyFloat_Check(BorrowedReference ob) => PyObject_TypeCheck(ob, PyFloatType); + internal static bool PyFloat_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyFloatType); /// /// Return value: New reference. @@ -1282,9 +1289,9 @@ internal static bool PyFloat_Check(BorrowedReference ob) // Python string API //==================================================================== internal static bool PyString_Check(BorrowedReference ob) - { - return PyObject_TYPE(ob) == PyStringType; - } + => PyObject_TypeCheck(ob, PyStringType); + internal static bool PyString_CheckExact(BorrowedReference ob) + => PyObject_TypeCheckExact(ob, PyStringType); internal static NewReference PyString_FromString(string value) { @@ -1643,6 +1650,8 @@ internal static bool PyType_IsSubtype(BorrowedReference t1, BorrowedReference t2 return Delegates.PyType_IsSubtype(t1, t2); } + internal static bool PyObject_TypeCheckExact(BorrowedReference ob, BorrowedReference tp) + => PyObject_TYPE(ob) == tp; internal static bool PyObject_TypeCheck(BorrowedReference ob, BorrowedReference tp) { BorrowedReference t = PyObject_TYPE(ob); From 782a0e5e01bd118043497e7327d09c355f101631 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Mon, 24 Oct 2022 09:56:23 -0700 Subject: [PATCH 16/34] allow decoders to override conversion of types derived from primitives when target type is System.Object useful to be able to change what numpy.float64 is converted to related to https://github.com/pythonnet/pythonnet/issues/1957 this is an alternative to https://github.com/pythonnet/pythonnet/pull/1958 --- src/embed_tests/Codecs.cs | 13 +++++++++++++ src/runtime/Converter.cs | 24 ++++++++++++++++++++---- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs index be416bc15..9b764d43f 100644 --- a/src/embed_tests/Codecs.cs +++ b/src/embed_tests/Codecs.cs @@ -355,6 +355,19 @@ from datetime import datetime scope.Exec("Codecs.AcceptsDateTime(datetime(2021, 1, 22))"); } + [Test] + public void FloatDerivedDecoded() + { + using var scope = Py.CreateScope(); + scope.Exec(@"class FloatDerived(float): pass"); + using var floatDerived = scope.Eval("FloatDerived"); + var decoder = new DecoderReturningPredefinedValue(floatDerived, 42); + PyObjectConversions.RegisterDecoder(decoder); + using var result = scope.Eval("FloatDerived()"); + object decoded = result.As(); + Assert.AreEqual(42, decoded); + } + [Test] public void ExceptionDecodedNoInstance() { diff --git a/src/runtime/Converter.cs b/src/runtime/Converter.cs index 3c46e9034..73bbd4a3a 100644 --- a/src/runtime/Converter.cs +++ b/src/runtime/Converter.cs @@ -361,28 +361,44 @@ internal static bool ToManagedValue(BorrowedReference value, Type obType, // conversions (Python string -> managed string). if (obType == objectType) { - if (Runtime.PyString_Check(value)) + if (Runtime.PyString_CheckExact(value)) { return ToPrimitive(value, stringType, out result, setError); } - if (Runtime.PyBool_Check(value)) + if (Runtime.PyBool_CheckExact(value)) { return ToPrimitive(value, boolType, out result, setError); } - if (Runtime.PyFloat_Check(value)) + if (Runtime.PyFloat_CheckExact(value)) { return ToPrimitive(value, doubleType, out result, setError); } - // give custom codecs a chance to take over conversion of ints and sequences + // give custom codecs a chance to take over conversion + // of ints, sequences, and types derived from primitives BorrowedReference pyType = Runtime.PyObject_TYPE(value); if (PyObjectConversions.TryDecode(value, pyType, obType, out result)) { return true; } + if (Runtime.PyString_Check(value)) + { + return ToPrimitive(value, stringType, out result, setError); + } + + if (Runtime.PyBool_Check(value)) + { + return ToPrimitive(value, boolType, out result, setError); + } + + if (Runtime.PyFloat_Check(value)) + { + return ToPrimitive(value, doubleType, out result, setError); + } + if (Runtime.PyInt_Check(value)) { result = new PyInt(value); From 2b5291000e0fae50bdf3db3e9a7a38157b319ba6 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 18 Sep 2022 16:03:40 +0200 Subject: [PATCH 17/34] Add Python 3.11 to metadata and workflows --- .github/workflows/main.yml | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2cc793621..ea93ce18c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,7 @@ jobs: fail-fast: false matrix: os: [windows, ubuntu, macos] - python: ["3.7", "3.8", "3.9", "3.10"] + python: ["3.7", "3.8", "3.9", "3.10", "3.11"] platform: [x64, x86] exclude: - os: ubuntu diff --git a/pyproject.toml b/pyproject.toml index 91f386fc6..d0512cc45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ classifiers = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", "Operating System :: MacOS :: MacOS X", From c0b4eb285ef3af1fdf90b0461e14c26b9969cf7a Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 19 Sep 2022 11:09:01 +0200 Subject: [PATCH 18/34] Improve geninterop script to handle new case in 3.11 --- tools/geninterop/geninterop.py | 144 +++++++++++++++++++-------------- 1 file changed, 83 insertions(+), 61 deletions(-) mode change 100644 => 100755 tools/geninterop/geninterop.py diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py old mode 100644 new mode 100755 index 0c80c1904..78e4d45c2 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -13,40 +13,26 @@ - clang """ -from __future__ import print_function - -import logging import os +import shutil import sys import sysconfig import subprocess -if sys.version_info.major > 2: - from io import StringIO -else: - from StringIO import StringIO - +from io import StringIO +from pathlib import Path from pycparser import c_ast, c_parser -_log = logging.getLogger() -logging.basicConfig(level=logging.DEBUG) - -PY_MAJOR = sys.version_info[0] -PY_MINOR = sys.version_info[1] - # rename some members from their C name when generating the C# _typeoffset_member_renames = { "ht_name": "name", - "ht_qualname": "qualname" + "ht_qualname": "qualname", + "getitem": "spec_cache_getitem", } def _check_output(*args, **kwargs): - """Check output wrapper for py2/py3 compatibility""" - output = subprocess.check_output(*args, **kwargs) - if PY_MAJOR == 2: - return output - return output.decode("ascii") + return subprocess.check_output(*args, **kwargs, encoding="utf8") class AstParser(object): @@ -92,7 +78,7 @@ def visit(self, node): self.visit_identifier(node) def visit_ast(self, ast): - for name, node in ast.children(): + for _name, node in ast.children(): self.visit(node) def visit_typedef(self, typedef): @@ -113,7 +99,7 @@ def visit_struct(self, struct): self.visit(decl) self._struct_members_stack.pop(0) self._struct_stack.pop(0) - elif self._ptr_decl_depth: + elif self._ptr_decl_depth or self._struct_members_stack: # the struct is empty, but add it as a member to the current # struct as the current member maybe a pointer to it. self._add_struct_member(struct.name) @@ -141,7 +127,8 @@ def _add_struct_member(self, type_name): current_struct = self._struct_stack[0] member_name = self._struct_members_stack[0] struct_members = self._struct_members.setdefault( - self._get_struct_name(current_struct), []) + self._get_struct_name(current_struct), [] + ) # get the node associated with this type node = None @@ -179,7 +166,6 @@ def _get_struct_name(self, node): class Writer(object): - def __init__(self): self._stream = StringIO() @@ -193,34 +179,47 @@ def to_string(self): return self._stream.getvalue() -def preprocess_python_headers(): +def preprocess_python_headers(*, cc=None, include_py=None): """Return Python.h pre-processed, ready for parsing. Requires clang. """ - fake_libc_include = os.path.join(os.path.dirname(__file__), - "fake_libc_include") + this_path = Path(__file__).parent + + fake_libc_include = this_path / "fake_libc_include" include_dirs = [fake_libc_include] - include_py = sysconfig.get_config_var("INCLUDEPY") + if cc is None: + cc = shutil.which("clang") + if cc is None: + cc = shutil.which("gcc") + if cc is None: + raise RuntimeError("No suitable C compiler found, need clang or gcc") + + if include_py is None: + include_py = sysconfig.get_config_var("INCLUDEPY") + include_py = Path(include_py) + include_dirs.append(include_py) - include_args = [c for p in include_dirs for c in ["-I", p]] + include_args = [c for p in include_dirs for c in ["-I", str(p)]] + # fmt: off defines = [ "-D", "__attribute__(x)=", "-D", "__inline__=inline", "-D", "__asm__=;#pragma asm", "-D", "__int64=long long", - "-D", "_POSIX_THREADS" + "-D", "_POSIX_THREADS", ] - if os.name == 'nt': + if sys.platform == "win32": defines.extend([ "-D", "__inline=inline", "-D", "__ptr32=", "-D", "__ptr64=", "-D", "__declspec(x)=", ]) + #fmt: on if hasattr(sys, "abiflags"): if "d" in sys.abiflags: @@ -228,8 +227,8 @@ def preprocess_python_headers(): if "u" in sys.abiflags: defines.extend(("-D", "PYTHON_WITH_WIDE_UNICODE")) - python_h = os.path.join(include_py, "Python.h") - cmd = ["clang", "-pthread"] + include_args + defines + ["-E", python_h] + python_h = include_py / "Python.h" + cmd = [cc, "-pthread"] + include_args + defines + ["-E", str(python_h)] # normalize as the parser doesn't like windows line endings. lines = [] @@ -240,16 +239,13 @@ def preprocess_python_headers(): return "\n".join(lines) - -def gen_interop_head(writer): +def gen_interop_head(writer, version, abi_flags): filename = os.path.basename(__file__) - abi_flags = getattr(sys, "abiflags", "").replace("m", "") - py_ver = "{0}.{1}".format(PY_MAJOR, PY_MINOR) - class_definition = """ -// Auto-generated by %s. + class_definition = f""" +// Auto-generated by {filename}. // DO NOT MODIFY BY HAND. -// Python %s: ABI flags: '%s' +// Python {".".join(version[:2])}: ABI flags: '{abi_flags}' // ReSharper disable InconsistentNaming // ReSharper disable IdentifierTypo @@ -261,7 +257,7 @@ def gen_interop_head(writer): using Python.Runtime.Native; namespace Python.Runtime -{""" % (filename, py_ver, abi_flags) +{{""" writer.extend(class_definition) @@ -271,25 +267,24 @@ def gen_interop_tail(writer): writer.extend(tail) -def gen_heap_type_members(parser, writer, type_name = None): +def gen_heap_type_members(parser, writer, type_name): """Generate the TypeOffset C# class""" members = parser.get_struct_members("PyHeapTypeObject") - type_name = type_name or "TypeOffset{0}{1}".format(PY_MAJOR, PY_MINOR) - class_definition = """ + class_definition = f""" [SuppressMessage("Style", "IDE1006:Naming Styles", Justification = "Following CPython", Scope = "type")] [StructLayout(LayoutKind.Sequential)] - internal class {0} : GeneratedTypeOffsets, ITypeOffsets + internal class {type_name} : GeneratedTypeOffsets, ITypeOffsets {{ - public {0}() {{ }} + public {type_name}() {{ }} // Auto-generated from PyHeapTypeObject in Python.h -""".format(type_name) +""" # All the members are sizeof(void*) so we don't need to do any # extra work to determine the size based on the type. - for name, tpy in members: + for name, _type in members: name = _typeoffset_member_renames.get(name, name) class_definition += " public int %s { get; private set; }\n" % name @@ -304,17 +299,18 @@ def gen_structure_code(parser, writer, type_name, indent): return False out = writer.append out(indent, "[StructLayout(LayoutKind.Sequential)]") - out(indent, "internal struct %s" % type_name) + out(indent, f"internal struct {type_name}") out(indent, "{") - for name, tpy in members: - out(indent + 1, "public IntPtr %s;" % name) + for name, _type in members: + out(indent + 1, f"public IntPtr {name};") out(indent, "}") out() return True -def main(): + +def main(*, cc=None, include_py=None, version=None, out=None): # preprocess Python.h and build the AST - python_h = preprocess_python_headers() + python_h = preprocess_python_headers(cc=cc, include_py=include_py) parser = c_parser.CParser() ast = parser.parse(python_h) @@ -323,21 +319,47 @@ def main(): ast_parser.visit(ast) writer = Writer() + + if include_py and not version: + raise RuntimeError("If the include path is overridden, version must be " + "defined" + ) + + if version: + version = version.split('.') + else: + version = sys.version_info + # generate the C# code - offsets_type_name = "NativeTypeOffset" if len(sys.argv) > 1 else None - gen_interop_head(writer) + abi_flags = getattr(sys, "abiflags", "").replace("m", "") + gen_interop_head(writer, version, abi_flags) - gen_heap_type_members(ast_parser, writer, type_name = offsets_type_name) + type_name = f"TypeOffset{version[0]}{version[1]}{abi_flags}" + gen_heap_type_members(ast_parser, writer, type_name) gen_interop_tail(writer) interop_cs = writer.to_string() - if len(sys.argv) > 1: - with open(sys.argv[1], "w") as fh: - fh.write(interop_cs) - else: + if not out or out == "-": print(interop_cs) + else: + with open(out, "w") as fh: + fh.write(interop_cs) if __name__ == "__main__": - sys.exit(main()) + import argparse + + a = argparse.ArgumentParser("Interop file generator for Python.NET") + a.add_argument("--cc", help="C compiler to use, either clang or gcc") + a.add_argument("--include-py", help="Include path of Python") + a.add_argument("--version", help="Python version") + a.add_argument("--out", help="Output path", default="-") + args = a.parse_args() + + sys.exit(main( + cc=args.cc, + include_py=args.include_py, + out=args.out, + version=args.version + )) From cc8606856278a511a2965c8224032cab40c6958d Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 19 Sep 2022 11:09:18 +0200 Subject: [PATCH 19/34] Fix offsets for 3.11 --- src/runtime/Native/TypeOffset311.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/Native/TypeOffset311.cs b/src/runtime/Native/TypeOffset311.cs index 7691236c9..de5afacb9 100644 --- a/src/runtime/Native/TypeOffset311.cs +++ b/src/runtime/Native/TypeOffset311.cs @@ -136,6 +136,6 @@ public TypeOffset311() { } public int ht_cached_keys { get; private set; } public int ht_module { get; private set; } public int _ht_tpname { get; private set; } + public int spec_cache_getitem { get; private set; } } } - From 5636262b27f883c484369c89ecdbcd54afa97a56 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sat, 15 Oct 2022 21:41:55 +0200 Subject: [PATCH 20/34] Update requires-python --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d0512cc45..52f1adb18 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "clr_loader>=0.2.2,<0.3.0" ] -requires-python = ">=3.7, <3.11" +requires-python = ">=3.7, <3.12" classifiers = [ "Development Status :: 5 - Production/Stable", From 8668579834623a1ef196840b8f2a93407f52701b Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sat, 15 Oct 2022 22:13:30 +0200 Subject: [PATCH 21/34] Define slots before initialization --- src/runtime/PythonTypes/PyType.cs | 1 + src/runtime/TypeManager.cs | 23 +++++++++++++---------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/runtime/PythonTypes/PyType.cs b/src/runtime/PythonTypes/PyType.cs index 260800592..af796a5c5 100644 --- a/src/runtime/PythonTypes/PyType.cs +++ b/src/runtime/PythonTypes/PyType.cs @@ -155,6 +155,7 @@ private static StolenReference FromSpec(TypeSpec spec, PyTuple? bases = null) using var nativeSpec = new NativeTypeSpec(spec); var basesRef = bases is null ? default : bases.Reference; var result = Runtime.PyType_FromSpecWithBases(in nativeSpec, basesRef); + // Runtime.PyErr_Print(); return result.StealOrThrow(); } } diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 217b4820e..6170f820f 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -475,17 +475,20 @@ internal static PyType CreateMetatypeWithGCHandleOffset() int size = Util.ReadInt32(Runtime.PyTypeType, TypeOffset.tp_basicsize) + IntPtr.Size // tp_clr_inst_offset ; - var result = new PyType(new TypeSpec("clr._internal.GCOffsetBase", basicSize: size, - new TypeSpec.Slot[] - { - - }, - TypeFlags.Default | TypeFlags.HeapType | TypeFlags.HaveGC), - bases: new PyTuple(new[] { py_type })); - SetRequiredSlots(result, seen: new HashSet()); - - Runtime.PyType_Modified(result); + var slots = new[] { + new TypeSpec.Slot(TypeSlotID.tp_traverse, subtype_traverse), + new TypeSpec.Slot(TypeSlotID.tp_clear, subtype_clear) + }; + var result = new PyType( + new TypeSpec( + "clr._internal.GCOffsetBase", + basicSize: size, + slots: slots, + TypeFlags.Default | TypeFlags.HeapType | TypeFlags.HaveGC + ), + bases: new PyTuple(new[] { py_type }) + ); return result; } From 218610ea33ef22ec1836f4fd43c3223676de5e51 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Fri, 28 Oct 2022 16:00:56 -0700 Subject: [PATCH 22/34] fixed positive PyInt converted to negative BigInteger BigInteger constructor uses the sign bit in the first byte. Since we explicitly handle the sign, the fix is to prepend a zero byte to the number, which does not change it, but ensures sign bit is zero. fixes https://github.com/pythonnet/pythonnet/issues/1990 --- CHANGELOG.md | 1 + src/embed_tests/TestPyInt.cs | 17 ++++++++++++----- src/runtime/PythonTypes/PyInt.cs | 2 +- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9781f289c..71c36b412 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Fixed - Fixed objects leaking when Python attached event handlers to them even if they were later removed +- Fixed `PyInt` conversion to `BigInteger` and `System.String` produced incorrect result for values between 128 and 255. ## [3.0.0](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.0) - 2022-09-29 diff --git a/src/embed_tests/TestPyInt.cs b/src/embed_tests/TestPyInt.cs index 822fe0715..c147e074b 100644 --- a/src/embed_tests/TestPyInt.cs +++ b/src/embed_tests/TestPyInt.cs @@ -191,16 +191,23 @@ public void ToBigInteger() { 0, 1, 2, 0x10, + 0x79, + 0x80, + 0x81, + 0xFF, 0x123, + 0x8000, 0x1234, + 0x8001, + 0x4000, + 0xFF, }; simpleValues = simpleValues.Concat(simpleValues.Select(v => -v)).ToArray(); - foreach (var val in simpleValues) - { - var pyInt = new PyInt(val); - Assert.AreEqual((BigInteger)val, pyInt.ToBigInteger()); - } + var expected = simpleValues.Select(v => new BigInteger(v)).ToArray(); + var actual = simpleValues.Select(v => new PyInt(v).ToBigInteger()).ToArray(); + + CollectionAssert.AreEqual(expected, actual); } [Test] diff --git a/src/runtime/PythonTypes/PyInt.cs b/src/runtime/PythonTypes/PyInt.cs index 6b3dbf210..e71462b74 100644 --- a/src/runtime/PythonTypes/PyInt.cs +++ b/src/runtime/PythonTypes/PyInt.cs @@ -212,7 +212,7 @@ public BigInteger ToBigInteger() offset++; neg = true; } - byte[] littleEndianBytes = new byte[(hex.Length - offset + 1) / 2]; + byte[] littleEndianBytes = new byte[(hex.Length - offset + 1) / 2 + 1]; for (; offset < hex.Length; offset++) { int littleEndianHexIndex = hex.Length - 1 - offset; From ddf5a701d2542c81b6bf0dcc6ad4a5a440713408 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 30 Oct 2022 21:36:03 +0100 Subject: [PATCH 23/34] Only clear dict if tp_dictoffset > 0 --- src/runtime/Types/ManagedType.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/runtime/Types/ManagedType.cs b/src/runtime/Types/ManagedType.cs index 2ed9d7970..97a19497c 100644 --- a/src/runtime/Types/ManagedType.cs +++ b/src/runtime/Types/ManagedType.cs @@ -148,8 +148,9 @@ protected static void ClearObjectDict(BorrowedReference ob) { BorrowedReference type = Runtime.PyObject_TYPE(ob); int instanceDictOffset = Util.ReadInt32(type, TypeOffset.tp_dictoffset); - Debug.Assert(instanceDictOffset > 0); - Runtime.Py_CLEAR(ob, instanceDictOffset); + // Debug.Assert(instanceDictOffset > 0); + if (instanceDictOffset > 0) + Runtime.Py_CLEAR(ob, instanceDictOffset); } protected static BorrowedReference GetObjectDict(BorrowedReference ob) From d3b56ffeeb35091365676362665460268d087a87 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 30 Oct 2022 21:40:26 +0100 Subject: [PATCH 24/34] Ensure that sub-processes in tests use the same runtime settings --- tests/conftest.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e61e3680e..1ac20e1dd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -79,9 +79,10 @@ def pytest_configure(config): ["dotnet", "publish", "-f", fw, "-o", str(bin_path), str(test_proj_path)] ) - from pythonnet import load - - load(runtime_opt, **runtime_params) + import os + os.environ["PYTHONNET_RUNTIME"] = runtime_opt + for k, v in runtime_params.items(): + os.environ[f"PYTHONNET_{runtime_opt.upper()}_{k.upper()}"] = v import clr From a6efeaee77404b75e557bae5cf795cbdd704bd00 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 30 Oct 2022 21:40:42 +0100 Subject: [PATCH 25/34] Update MaxSupportedVersion --- src/runtime/PythonEngine.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/PythonEngine.cs b/src/runtime/PythonEngine.cs index e5879ae67..ddf597c4f 100644 --- a/src/runtime/PythonEngine.cs +++ b/src/runtime/PythonEngine.cs @@ -128,7 +128,7 @@ public static string PythonPath } public static Version MinSupportedVersion => new(3, 7); - public static Version MaxSupportedVersion => new(3, 10, int.MaxValue, int.MaxValue); + public static Version MaxSupportedVersion => new(3, 11, int.MaxValue, int.MaxValue); public static bool IsSupportedVersion(Version version) => version >= MinSupportedVersion && version <= MaxSupportedVersion; public static string Version From da082ac487190aed5d325a346b0a6c268c870020 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Sun, 30 Oct 2022 23:16:57 +0100 Subject: [PATCH 26/34] Enforce tp_traverse/clear in AllocateTypeObject --- src/runtime/TypeManager.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index 6170f820f..e0a78ba49 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -620,6 +620,11 @@ internal static PyType AllocateTypeObject(string name, PyType metatype) Util.WriteRef(type, TypeOffset.name, new NewReference(temp).Steal()); Util.WriteRef(type, TypeOffset.qualname, temp.Steal()); + // Ensure that tp_traverse and tp_clear are always set, since their + // existence is enforced in newer Python versions in PyType_Ready + Util.WriteIntPtr(type, TypeOffset.tp_traverse, subtype_traverse); + Util.WriteIntPtr(type, TypeOffset.tp_clear, subtype_clear); + InheritSubstructs(type.Reference.DangerousGetAddress()); return type; From e9283e3ec35184c7c4464540756f3d20c1301bfd Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 31 Oct 2022 14:51:18 +0100 Subject: [PATCH 27/34] Ensure that Python is initialized before probing properties --- src/embed_tests/TestPythonEngineProperties.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/embed_tests/TestPythonEngineProperties.cs b/src/embed_tests/TestPythonEngineProperties.cs index ca9164a1d..bbaff047f 100644 --- a/src/embed_tests/TestPythonEngineProperties.cs +++ b/src/embed_tests/TestPythonEngineProperties.cs @@ -9,6 +9,7 @@ public class TestPythonEngineProperties [Test] public static void GetBuildinfoDoesntCrash() { + PythonEngine.Initialize(); using (Py.GIL()) { string s = PythonEngine.BuildInfo; @@ -21,6 +22,7 @@ public static void GetBuildinfoDoesntCrash() [Test] public static void GetCompilerDoesntCrash() { + PythonEngine.Initialize(); using (Py.GIL()) { string s = PythonEngine.Compiler; @@ -34,6 +36,7 @@ public static void GetCompilerDoesntCrash() [Test] public static void GetCopyrightDoesntCrash() { + PythonEngine.Initialize(); using (Py.GIL()) { string s = PythonEngine.Copyright; @@ -46,6 +49,7 @@ public static void GetCopyrightDoesntCrash() [Test] public static void GetPlatformDoesntCrash() { + PythonEngine.Initialize(); using (Py.GIL()) { string s = PythonEngine.Platform; @@ -58,6 +62,7 @@ public static void GetPlatformDoesntCrash() [Test] public static void GetVersionDoesntCrash() { + PythonEngine.Initialize(); using (Py.GIL()) { string s = PythonEngine.Version; From cc97b8a49646e952a774e105f77d00f21761d864 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Tue, 1 Nov 2022 14:46:39 +0100 Subject: [PATCH 28/34] Add an Action variant of TryUsingDll --- src/runtime/Runtime.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 6238119ff..26e83a5f9 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -672,6 +672,9 @@ internal static unsafe nint Refcount(BorrowedReference op) [Pure] internal static int Refcount32(BorrowedReference op) => checked((int)Refcount(op)); + internal static void TryUsingDll(Action op) => + TryUsingDll(() => { op(); return 0; }); + /// /// Call specified function, and handle PythonDLL-related failures. /// From 096f50a21e235b6a8a15c011d0a390468901daa2 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 2 Nov 2022 06:36:34 +0100 Subject: [PATCH 29/34] Adjust code a bit and skip PythonHome tests for empty strings --- src/embed_tests/TestPythonEngineProperties.cs | 49 ++++++++++++------- src/runtime/PythonEngine.cs | 15 ++++-- 2 files changed, 41 insertions(+), 23 deletions(-) diff --git a/src/embed_tests/TestPythonEngineProperties.cs b/src/embed_tests/TestPythonEngineProperties.cs index bbaff047f..be91d7f45 100644 --- a/src/embed_tests/TestPythonEngineProperties.cs +++ b/src/embed_tests/TestPythonEngineProperties.cs @@ -96,9 +96,6 @@ public static void GetProgramNameDefault() /// Test default behavior of PYTHONHOME. If ENVVAR is set it will /// return the same value. If not, returns EmptyString. /// - /// - /// AppVeyor.yml has been update to tests with ENVVAR set. - /// [Test] public static void GetPythonHomeDefault() { @@ -114,22 +111,19 @@ public static void GetPythonHomeDefault() [Test] public void SetPythonHome() { - // We needs to ensure that engine was started and shutdown at least once before setting dummy home. - // Otherwise engine will not run with dummy path with random problem. - if (!PythonEngine.IsInitialized) - { - PythonEngine.Initialize(); - } - + PythonEngine.Initialize(); + var pythonHomeBackup = PythonEngine.PythonHome; PythonEngine.Shutdown(); - var pythonHomeBackup = PythonEngine.PythonHome; + if (pythonHomeBackup == "") + Assert.Inconclusive("Can't reset PythonHome to empty string, skipping"); var pythonHome = "/dummypath/"; PythonEngine.PythonHome = pythonHome; PythonEngine.Initialize(); + Assert.AreEqual(pythonHome, PythonEngine.PythonHome); PythonEngine.Shutdown(); // Restoring valid pythonhome. @@ -139,15 +133,12 @@ public void SetPythonHome() [Test] public void SetPythonHomeTwice() { - // We needs to ensure that engine was started and shutdown at least once before setting dummy home. - // Otherwise engine will not run with dummy path with random problem. - if (!PythonEngine.IsInitialized) - { - PythonEngine.Initialize(); - } + PythonEngine.Initialize(); + var pythonHomeBackup = PythonEngine.PythonHome; PythonEngine.Shutdown(); - var pythonHomeBackup = PythonEngine.PythonHome; + if (pythonHomeBackup == "") + Assert.Inconclusive("Can't reset PythonHome to empty string, skipping"); var pythonHome = "/dummypath/"; @@ -161,6 +152,26 @@ public void SetPythonHomeTwice() PythonEngine.PythonHome = pythonHomeBackup; } + [Test] + [Ignore("Currently buggy in Python")] + public void SetPythonHomeEmptyString() + { + PythonEngine.Initialize(); + + var backup = PythonEngine.PythonHome; + if (backup == "") + { + PythonEngine.Shutdown(); + Assert.Inconclusive("Can't reset PythonHome to empty string, skipping"); + } + PythonEngine.PythonHome = ""; + + Assert.AreEqual("", PythonEngine.PythonHome); + + PythonEngine.PythonHome = backup; + PythonEngine.Shutdown(); + } + [Test] public void SetProgramName() { @@ -207,7 +218,7 @@ public void SetPythonPath() // The list sys.path is initialized with this value on interpreter startup; // it can be (and usually is) modified later to change the search path for loading modules. // See https://docs.python.org/3/c-api/init.html#c.Py_GetPath - // After PythonPath is set, then PythonEngine.PythonPath will correctly return the full search path. + // After PythonPath is set, then PythonEngine.PythonPath will correctly return the full search path. PythonEngine.Shutdown(); diff --git a/src/runtime/PythonEngine.cs b/src/runtime/PythonEngine.cs index ddf597c4f..4ed45b9e9 100644 --- a/src/runtime/PythonEngine.cs +++ b/src/runtime/PythonEngine.cs @@ -47,6 +47,14 @@ public static bool IsInitialized get { return initialized; } } + private static void EnsureInitialized() + { + if (!IsInitialized) + throw new InvalidOperationException( + "Python must be initialized for this operation" + ); + } + /// Set to true to enable GIL debugging assistance. public static bool DebugGIL { get; set; } = false; @@ -96,6 +104,7 @@ public static string PythonHome { get { + EnsureInitialized(); IntPtr p = Runtime.TryUsingDll(() => Runtime.Py_GetPythonHome()); return UcsMarshaler.PtrToPy3UnicodePy2String(p) ?? ""; } @@ -103,10 +112,8 @@ public static string PythonHome { // this value is null in the beginning Marshal.FreeHGlobal(_pythonHome); - _pythonHome = Runtime.TryUsingDll( - () => UcsMarshaler.Py3UnicodePy2StringtoPtr(value) - ); - Runtime.Py_SetPythonHome(_pythonHome); + _pythonHome = UcsMarshaler.Py3UnicodePy2StringtoPtr(value); + Runtime.TryUsingDll(() => Runtime.Py_SetPythonHome(_pythonHome)); } } From 461c9c708ce36a054c363700f232a1b5df09ed2e Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 2 Nov 2022 07:59:03 +0100 Subject: [PATCH 30/34] Add tests for deriving a Python subclass from a generic interface --- src/testing/interfacetest.cs | 23 ++++++++++++++++++++++- tests/test_subclass.py | 23 ++++++++++++++++++++++- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/testing/interfacetest.cs b/src/testing/interfacetest.cs index 7c5d937b9..a1a7106b5 100644 --- a/src/testing/interfacetest.cs +++ b/src/testing/interfacetest.cs @@ -79,7 +79,7 @@ private interface IPrivate { } } - + public interface IOutArg { string MyMethod_Out(string name, out int index); @@ -93,4 +93,25 @@ public static int CallMyMethod_Out(IOutArg myInterface) return index; } } + + public interface IGenericInterface + { + public T Get(T x); + } + + public class SpecificInterfaceUser + { + public SpecificInterfaceUser(IGenericInterface some, int x) + { + some.Get(x); + } + } + + public class GenericInterfaceUser + { + public GenericInterfaceUser(IGenericInterface some, T x) + { + some.Get(x); + } + } } diff --git a/tests/test_subclass.py b/tests/test_subclass.py index fa82c3663..a51e89da3 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -9,7 +9,7 @@ import System import pytest from Python.Test import (IInterfaceTest, SubClassTest, EventArgsTest, - FunctionsTest) + FunctionsTest, IGenericInterface) from System.Collections.Generic import List @@ -29,6 +29,17 @@ def bar(self, x, i): return InterfaceTestClass +def interface_generic_class_fixture(subnamespace): + + class GenericInterfaceImpl(IGenericInterface[int]): + __namespace__ = "Python.Test." + subnamespace + + def Get(self, x): + return x + + return GenericInterfaceImpl + + def derived_class_fixture(subnamespace): """Delay creation of class until test starts.""" @@ -306,3 +317,13 @@ class Derived(BaseClass): import gc gc.collect() + +def test_generic_interface(): + from System import Int32 + from Python.Test import GenericInterfaceUser, SpecificInterfaceUser + + GenericInterfaceImpl = interface_generic_class_fixture(test_generic_interface.__name__) + + obj = GenericInterfaceImpl() + SpecificInterfaceUser(obj, Int32(0)) + GenericInterfaceUser[Int32](obj, Int32(0)) From 15e2e9596ccf6c23ea1bc27db4977055455c170d Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 2 Nov 2022 08:29:02 +0100 Subject: [PATCH 31/34] Set PYTHONHOME for tests --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ea93ce18c..93963c70a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,15 +54,17 @@ jobs: run: | pip install -v . - - name: Set Python DLL path (non Windows) + - name: Set Python DLL path and PYTHONHOME (non Windows) if: ${{ matrix.os != 'windows' }} run: | echo PYTHONNET_PYDLL=$(python -m find_libpython) >> $GITHUB_ENV + echo PYTHONHOME=$(python -c 'import sys; print(sys.prefix)') >> $GITHUB_ENV - - name: Set Python DLL path (Windows) + - name: Set Python DLL path and PYTHONHOME (Windows) if: ${{ matrix.os == 'windows' }} run: | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -InputObject "PYTHONNET_PYDLL=$(python -m find_libpython)" + Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append -InputObject "PYTHONHOME=$(python -c 'import sys; print(sys.prefix)')" - name: Embedding tests run: dotnet test --runtime any-${{ matrix.platform }} --logger "console;verbosity=detailed" src/embed_tests/ From cc364c359beac37b218cd302af3b600f29e17b35 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 2 Nov 2022 09:08:32 +0100 Subject: [PATCH 32/34] Update changelog and add Python 3.11 to metadata --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71c36b412..8ac8e6c06 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added +- Support for Python 3.11 + ### Changed ### Fixed From 5c1e02f0b5d38def43e8547b5ca701d724f80c80 Mon Sep 17 00:00:00 2001 From: Victor Nova Date: Wed, 2 Nov 2022 11:15:04 -0700 Subject: [PATCH 33/34] fixed resolution of generic methods in Python implementations since RuntimeMethodHandle does not encode generic arguments, I had to supply RuntimeTypeHandle of the declaring type to be able to get fully specified method --- src/runtime/Types/ClassDerived.cs | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 02288faee..cf6d9b16b 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -436,6 +436,7 @@ private static void AddVirtualMethod(MethodInfo method, Type baseType, TypeBuild il.Emit(OpCodes.Ldloc_0); il.Emit(OpCodes.Ldtoken, method); + il.Emit(OpCodes.Ldtoken, method.DeclaringType); #pragma warning disable CS0618 // PythonDerivedType is for internal use only if (method.ReturnType == typeof(void)) { @@ -505,6 +506,7 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde il.DeclareLocal(typeof(object[])); il.DeclareLocal(typeof(RuntimeMethodHandle)); + il.DeclareLocal(typeof(RuntimeTypeHandle)); // this il.Emit(OpCodes.Ldarg_0); @@ -546,6 +548,11 @@ private static void AddPythonMethod(string methodName, PyObject func, TypeBuilde il.Emit(OpCodes.Ldloca_S, 1); il.Emit(OpCodes.Initobj, typeof(RuntimeMethodHandle)); il.Emit(OpCodes.Ldloc_1); + + // type handle is also not required + il.Emit(OpCodes.Ldloca_S, 2); + il.Emit(OpCodes.Initobj, typeof(RuntimeTypeHandle)); + il.Emit(OpCodes.Ldloc_2); #pragma warning disable CS0618 // PythonDerivedType is for internal use only // invoke the method @@ -698,7 +705,7 @@ public class PythonDerivedType /// class) it calls it, otherwise it calls the base method. /// public static T? InvokeMethod(IPythonDerivedType obj, string methodName, string origMethodName, - object[] args, RuntimeMethodHandle methodHandle) + object[] args, RuntimeMethodHandle methodHandle, RuntimeTypeHandle declaringTypeHandle) { var self = GetPyObj(obj); @@ -724,7 +731,10 @@ public class PythonDerivedType } PyObject py_result = method.Invoke(pyargs); - PyTuple? result_tuple = MarshalByRefsBack(args, methodHandle, py_result, outsOffset: 1); + var clrMethod = methodHandle != default + ? MethodBase.GetMethodFromHandle(methodHandle, declaringTypeHandle) + : null; + PyTuple? result_tuple = MarshalByRefsBack(args, clrMethod, py_result, outsOffset: 1); return result_tuple is not null ? result_tuple[0].As() : py_result.As(); @@ -754,7 +764,7 @@ public class PythonDerivedType } public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, string origMethodName, - object?[] args, RuntimeMethodHandle methodHandle) + object?[] args, RuntimeMethodHandle methodHandle, RuntimeTypeHandle declaringTypeHandle) { var self = GetPyObj(obj); if (null != self.Ref) @@ -779,7 +789,10 @@ public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, s } PyObject py_result = method.Invoke(pyargs); - MarshalByRefsBack(args, methodHandle, py_result, outsOffset: 0); + var clrMethod = methodHandle != default + ? MethodBase.GetMethodFromHandle(methodHandle, declaringTypeHandle) + : null; + MarshalByRefsBack(args, clrMethod, py_result, outsOffset: 0); return; } } @@ -811,12 +824,11 @@ public static void InvokeMethodVoid(IPythonDerivedType obj, string methodName, s /// as a tuple of new values for those arguments, and updates corresponding /// elements of array. /// - private static PyTuple? MarshalByRefsBack(object?[] args, RuntimeMethodHandle methodHandle, PyObject pyResult, int outsOffset) + private static PyTuple? MarshalByRefsBack(object?[] args, MethodBase? method, PyObject pyResult, int outsOffset) { - if (methodHandle == default) return null; + if (method is null) return null; - var originalMethod = MethodBase.GetMethodFromHandle(methodHandle); - var parameters = originalMethod.GetParameters(); + var parameters = method.GetParameters(); PyTuple? outs = null; int byrefIndex = 0; for (int i = 0; i < parameters.Length; ++i) From e28dd80c3a8fb60bd62671b765ff41c98ac5b1e9 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Thu, 3 Nov 2022 00:21:39 +0100 Subject: [PATCH 34/34] Release 3.0.1 --- CHANGELOG.md | 11 +++++++++++ version.txt | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ac8e6c06..13bf09c2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,14 +9,25 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ### Added +### Changed + +### Fixed + +## [3.0.1](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.1) - 2022-11-03 + +### Added + - Support for Python 3.11 ### Changed +- Allow decoders to override conversion of types derived from primitive types + ### Fixed - Fixed objects leaking when Python attached event handlers to them even if they were later removed - Fixed `PyInt` conversion to `BigInteger` and `System.String` produced incorrect result for values between 128 and 255. +- Fixed implementing a generic interface with a Python class ## [3.0.0](https://github.com/pythonnet/pythonnet/releases/tag/v3.0.0) - 2022-09-29 diff --git a/version.txt b/version.txt index 0f9d6b15d..cb2b00e4f 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.1.0-dev +3.0.1