From 2eb143fe0301a825427f99206310e9aafd47d098 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 11 Oct 2021 15:59:44 +0200 Subject: [PATCH 1/4] Fix runtime loading error message --- pythonnet/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pythonnet/__init__.py b/pythonnet/__init__.py index 188980b8b..7eec90f27 100644 --- a/pythonnet/__init__.py +++ b/pythonnet/__init__.py @@ -1,4 +1,3 @@ -import os import sys import clr_loader @@ -11,7 +10,7 @@ def set_runtime(runtime): global _RUNTIME if _LOADED: - raise RuntimeError("The runtime {runtime} has already been loaded".format(_RUNTIME)) + raise RuntimeError("The runtime {} has already been loaded".format(_RUNTIME)) _RUNTIME = runtime From e5b0a07b839c6da220369eca5b0262f60acafe69 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 11 Oct 2021 09:36:12 +0200 Subject: [PATCH 2/4] Add Python 3.10 support - Add Python 3.10 typeoffsets - Replace call to (removed) `PyUnicode_GetMax` by platform detection, as AFAIK only Windows uses 16bit wchar_t --- src/runtime/CustomMarshaler.cs | 2 +- src/runtime/interop310.cs | 140 +++++++++++++++++++++++++++++++++ src/runtime/runtime.cs | 10 --- 3 files changed, 141 insertions(+), 11 deletions(-) create mode 100644 src/runtime/interop310.cs diff --git a/src/runtime/CustomMarshaler.cs b/src/runtime/CustomMarshaler.cs index 3ef5cd662..4ba603609 100644 --- a/src/runtime/CustomMarshaler.cs +++ b/src/runtime/CustomMarshaler.cs @@ -41,7 +41,7 @@ public int GetNativeDataSize() /// internal class UcsMarshaler : MarshalerBase { - internal static readonly int _UCS = Runtime.PyUnicode_GetMax() <= 0xFFFF ? 2 : 4; + internal static readonly int _UCS = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? 2 : 4; internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; private static readonly MarshalerBase Instance = new UcsMarshaler(); diff --git a/src/runtime/interop310.cs b/src/runtime/interop310.cs new file mode 100644 index 000000000..fc0ca1e8c --- /dev/null +++ b/src/runtime/interop310.cs @@ -0,0 +1,140 @@ + +// Auto-generated by geninterop.py. +// DO NOT MODIFY BY HAND. + +// Python 3.10: ABI flags: '' + +// ReSharper disable InconsistentNaming +// ReSharper disable IdentifierTypo + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +using Python.Runtime.Native; + +namespace Python.Runtime +{ + [SuppressMessage("Style", "IDE1006:Naming Styles", + Justification = "Following CPython", + Scope = "type")] + + [StructLayout(LayoutKind.Sequential)] + internal class TypeOffset310 : GeneratedTypeOffsets, ITypeOffsets + { + public TypeOffset310() { } + // Auto-generated from PyHeapTypeObject in Python.h + public int ob_refcnt { get; private set; } + public int ob_type { get; private set; } + public int ob_size { get; private set; } + public int tp_name { get; private set; } + public int tp_basicsize { get; private set; } + public int tp_itemsize { get; private set; } + public int tp_dealloc { get; private set; } + public int tp_vectorcall_offset { get; private set; } + public int tp_getattr { get; private set; } + public int tp_setattr { get; private set; } + public int tp_as_async { get; private set; } + public int tp_repr { get; private set; } + public int tp_as_number { get; private set; } + public int tp_as_sequence { get; private set; } + public int tp_as_mapping { get; private set; } + public int tp_hash { get; private set; } + public int tp_call { get; private set; } + public int tp_str { get; private set; } + public int tp_getattro { get; private set; } + public int tp_setattro { get; private set; } + public int tp_as_buffer { get; private set; } + public int tp_flags { get; private set; } + public int tp_doc { get; private set; } + public int tp_traverse { get; private set; } + public int tp_clear { get; private set; } + public int tp_richcompare { get; private set; } + public int tp_weaklistoffset { get; private set; } + public int tp_iter { get; private set; } + public int tp_iternext { get; private set; } + public int tp_methods { get; private set; } + public int tp_members { get; private set; } + public int tp_getset { get; private set; } + public int tp_base { get; private set; } + public int tp_dict { get; private set; } + public int tp_descr_get { get; private set; } + public int tp_descr_set { get; private set; } + public int tp_dictoffset { get; private set; } + public int tp_init { get; private set; } + public int tp_alloc { get; private set; } + public int tp_new { get; private set; } + public int tp_free { get; private set; } + public int tp_is_gc { get; private set; } + public int tp_bases { get; private set; } + public int tp_mro { get; private set; } + public int tp_cache { get; private set; } + public int tp_subclasses { get; private set; } + public int tp_weaklist { get; private set; } + public int tp_del { get; private set; } + public int tp_version_tag { get; private set; } + public int tp_finalize { get; private set; } + public int tp_vectorcall { get; private set; } + public int am_await { get; private set; } + public int am_aiter { get; private set; } + public int am_anext { get; private set; } + public int am_send { get; private set; } + public int nb_add { get; private set; } + public int nb_subtract { get; private set; } + public int nb_multiply { get; private set; } + public int nb_remainder { get; private set; } + public int nb_divmod { get; private set; } + public int nb_power { get; private set; } + public int nb_negative { get; private set; } + public int nb_positive { get; private set; } + public int nb_absolute { get; private set; } + public int nb_bool { get; private set; } + public int nb_invert { get; private set; } + public int nb_lshift { get; private set; } + public int nb_rshift { get; private set; } + public int nb_and { get; private set; } + public int nb_xor { get; private set; } + public int nb_or { get; private set; } + public int nb_int { get; private set; } + public int nb_reserved { get; private set; } + public int nb_float { get; private set; } + public int nb_inplace_add { get; private set; } + public int nb_inplace_subtract { get; private set; } + public int nb_inplace_multiply { get; private set; } + public int nb_inplace_remainder { get; private set; } + public int nb_inplace_power { get; private set; } + public int nb_inplace_lshift { get; private set; } + public int nb_inplace_rshift { get; private set; } + public int nb_inplace_and { get; private set; } + public int nb_inplace_xor { get; private set; } + public int nb_inplace_or { get; private set; } + public int nb_floor_divide { get; private set; } + public int nb_true_divide { get; private set; } + public int nb_inplace_floor_divide { get; private set; } + public int nb_inplace_true_divide { get; private set; } + public int nb_index { get; private set; } + public int nb_matrix_multiply { get; private set; } + public int nb_inplace_matrix_multiply { get; private set; } + public int mp_length { get; private set; } + public int mp_subscript { get; private set; } + public int mp_ass_subscript { get; private set; } + public int sq_length { get; private set; } + public int sq_concat { get; private set; } + public int sq_repeat { get; private set; } + public int sq_item { get; private set; } + public int was_sq_slice { get; private set; } + public int sq_ass_item { get; private set; } + public int was_sq_ass_slice { get; private set; } + public int sq_contains { get; private set; } + public int sq_inplace_concat { get; private set; } + public int sq_inplace_repeat { get; private set; } + public int bf_getbuffer { get; private set; } + public int bf_releasebuffer { get; private set; } + public int name { get; private set; } + public int ht_slots { get; private set; } + public int qualname { get; private set; } + public int ht_cached_keys { get; private set; } + public int ht_module { get; private set; } + } +} + diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index d2653a510..b4b045b4a 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -36,12 +36,6 @@ private static string GetDefaultDllName() string dll = Environment.GetEnvironmentVariable("PYTHONNET_PYDLL"); if (dll is not null) return dll; - try - { - LibraryLoader.Instance.GetFunction(IntPtr.Zero, "PyUnicode_GetMax"); - return null; - } catch (MissingMethodException) { } - string verString = Environment.GetEnvironmentVariable("PYTHONNET_PYVER"); if (!Version.TryParse(verString, out var version)) return null; @@ -1581,8 +1575,6 @@ internal static bool PyUnicode_Check(IntPtr ob) internal static IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err) => Delegates.PyUnicode_FromEncodedObject(ob, enc, err); - internal static int PyUnicode_GetMax() => Delegates.PyUnicode_GetMax(); - internal static long PyUnicode_GetSize(IntPtr ob) { return (long)_PyUnicode_GetSize(ob); @@ -2462,7 +2454,6 @@ static Delegates() PyUnicode_FromObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromObject), GetUnmanagedDll(_PythonDll)); PyUnicode_DecodeUTF16 = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_DecodeUTF16), GetUnmanagedDll(_PythonDll)); PyUnicode_FromEncodedObject = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_FromEncodedObject), GetUnmanagedDll(_PythonDll)); - PyUnicode_GetMax = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_GetMax), GetUnmanagedDll(_PythonDll)); _PyUnicode_GetSize = (delegate* unmanaged[Cdecl])GetFunctionByName("PyUnicode_GetSize", GetUnmanagedDll(_PythonDll)); PyUnicode_AsUnicode = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUnicode), GetUnmanagedDll(_PythonDll)); PyUnicode_AsUTF16String = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(PyUnicode_AsUTF16String), GetUnmanagedDll(_PythonDll)); @@ -2753,7 +2744,6 @@ static Delegates() internal static delegate* unmanaged[Cdecl] PyUnicode_FromObject { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_FromEncodedObject { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_DecodeUTF16 { get; } - internal static delegate* unmanaged[Cdecl] PyUnicode_GetMax { get; } internal static delegate* unmanaged[Cdecl] _PyUnicode_GetSize { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_AsUnicode { get; } internal static delegate* unmanaged[Cdecl] PyUnicode_AsUTF16String { get; } From 78a22e672e1ee218e47dc993b10d329cf121ba59 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 11 Oct 2021 09:37:42 +0200 Subject: [PATCH 3/4] Update CI configuration for Python 3.10 Also increase CI timeout for now as some dependencies have no wheels available, yet, and are thus compiled during CI. --- .github/workflows/main.yml | 4 ++-- appveyor.yml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e33a53b84..61920bd34 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,13 +6,13 @@ jobs: build-test: name: Build and Test runs-on: ${{ matrix.os }}-latest - timeout-minutes: 7 + timeout-minutes: 15 strategy: fail-fast: false matrix: os: [windows, ubuntu, macos] - python: ["3.6", "3.7", "3.8", "3.9"] + python: ["3.6", "3.7", "3.8", "3.9", "3.10"] platform: [x64] shutdown_mode: [Normal, Soft] diff --git a/appveyor.yml b/appveyor.yml index cc3815c62..6b4009110 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,10 +15,13 @@ environment: CODECOV_ENV: PYTHON_VERSION, PLATFORM matrix: + - PYTHON_VERSION: 3.10 - PYTHON_VERSION: 3.9 - PYTHON_VERSION: 3.8 - PYTHON_VERSION: 3.7 - PYTHON_VERSION: 3.6 + - PYTHON_VERSION: 3.10 + PYTHONNET_SHUTDOWN_MODE: Soft - PYTHON_VERSION: 3.9 PYTHONNET_SHUTDOWN_MODE: Soft - PYTHON_VERSION: 3.8 From d059a53c35197032bf11c29a72be29460ce60c77 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Mon, 11 Oct 2021 10:19:58 +0200 Subject: [PATCH 4/4] Drop Appveyor CI --- appveyor.yml | 59 ------------------- ci/appveyor_run_tests.ps1 | 120 -------------------------------------- 2 files changed, 179 deletions(-) delete mode 100644 appveyor.yml delete mode 100644 ci/appveyor_run_tests.ps1 diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 6b4009110..000000000 --- a/appveyor.yml +++ /dev/null @@ -1,59 +0,0 @@ -version: '{branch}-{build}' -build: off - -image: - - Visual Studio 2019 - -platform: - - x86 - - x64 - -environment: - global: - PYTHONUNBUFFERED: 'True' - PYTHONWARNINGS: 'ignore:::wheel.pep425tags:' - CODECOV_ENV: PYTHON_VERSION, PLATFORM - - matrix: - - PYTHON_VERSION: 3.10 - - PYTHON_VERSION: 3.9 - - PYTHON_VERSION: 3.8 - - PYTHON_VERSION: 3.7 - - PYTHON_VERSION: 3.6 - - PYTHON_VERSION: 3.10 - PYTHONNET_SHUTDOWN_MODE: Soft - - PYTHON_VERSION: 3.9 - PYTHONNET_SHUTDOWN_MODE: Soft - - PYTHON_VERSION: 3.8 - PYTHONNET_SHUTDOWN_MODE: Soft - - PYTHON_VERSION: 3.7 - PYTHONNET_SHUTDOWN_MODE: Soft - - PYTHON_VERSION: 3.6 - PYTHONNET_SHUTDOWN_MODE: Soft -init: - # Update Environment Variables based on matrix/platform - - set PY_VER=%PYTHON_VERSION:.=% - - set PYTHON=C:\PYTHON%PY_VER% - - if %PLATFORM%==x64 (set PYTHON=%PYTHON%-x64) - - set PYTHONNET_PYDLL=%PYTHON%\python%PY_VER%.dll - - # Put desired Python version first in PATH - - set PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% - -install: - - python -m pip install -U pip - - pip install --upgrade -r requirements.txt --quiet - -build_script: - # Create clean `sdist`. Only used for releases - - python setup.py --quiet sdist - - python setup.py bdist_wheel - -test_script: - - pip install --no-index --find-links=.\dist\ pythonnet - #- ps: .\ci\appveyor_run_tests.ps1 - - pytest - - dotnet test src/embed_tests/ - -artifacts: - - path: dist\* diff --git a/ci/appveyor_run_tests.ps1 b/ci/appveyor_run_tests.ps1 deleted file mode 100644 index 7d35131f4..000000000 --- a/ci/appveyor_run_tests.ps1 +++ /dev/null @@ -1,120 +0,0 @@ -# Script to simplify AppVeyor configuration and resolve path to tools - -$stopwatch = [Diagnostics.Stopwatch]::StartNew() -[array]$timings = @() - -# Test Runner framework being used for embedded tests -$CS_RUNNER = "nunit3-console" - -$XPLAT = $env:BUILD_OPTS -eq "--xplat" - -# Needed for ARCH specific runners(NUnit2/XUnit3). Skip for NUnit3 -if ($FALSE -and $env:PLATFORM -eq "x86"){ - $CS_RUNNER = $CS_RUNNER + "-x86" -} - -# Executable paths for OpenCover -# Note if OpenCover fails, it won't affect the exit codes. -$OPENCOVER = Resolve-Path .\packages\OpenCover.*\tools\OpenCover.Console.exe -if ($XPLAT){ - $CS_RUNNER = Resolve-Path $env:USERPROFILE\.nuget\packages\nunit.consolerunner\*\tools\"$CS_RUNNER".exe -} -else{ - $CS_RUNNER = Resolve-Path .\packages\NUnit.*\tools\"$CS_RUNNER".exe -} -$PY = Get-Command python - -# Can't use ".\build\*\Python.EmbeddingTest.dll". Missing framework files. -$CS_TESTS = ".\src\embed_tests\bin\Python.EmbeddingTest.dll" -$RUNTIME_DIR = ".\src\runtime\bin\" - -function ReportTime { - param([string] $action) - - $timeSpent = $stopwatch.Elapsed - $timings += [pscustomobject]@{action=$action; timeSpent=$timeSpent} - Write-Host $action " in " $timeSpent -ForegroundColor "Green" - $stopwatch.Restart() -} - -ReportTime "Preparation done" - -# Run python tests with C# coverage -Write-Host ("Starting Python tests") -ForegroundColor "Green" -.$OPENCOVER -register:user -searchdirs:"$RUNTIME_DIR" -output:py.coverage ` - -target:"$PY" -targetargs:"-m pytest" ` - -returntargetcode -$PYTHON_STATUS = $LastExitCode -if ($PYTHON_STATUS -ne 0) { - Write-Host "Python tests failed, continuing to embedded tests" -ForegroundColor "Red" - ReportTime "" -} else { - ReportTime "Python tests completed" -} - -# Run Embedded tests with C# coverage -Write-Host ("Starting embedded tests") -ForegroundColor "Green" -.$OPENCOVER -register:user -searchdirs:"$RUNTIME_DIR" -output:cs.coverage ` - -target:"$CS_RUNNER" -targetargs:"$CS_TESTS --labels=All" ` - -filter:"+[*]Python.Runtime*" ` - -returntargetcode -$CS_STATUS = $LastExitCode -if ($CS_STATUS -ne 0) { - Write-Host "Embedded tests failed" -ForegroundColor "Red" - ReportTime "" -} else { - ReportTime "Embedded tests completed" - - # NuGet for pythonnet-2.3 only has 64-bit binary for Python 3.5 - # the test is only built using modern stack - if (($env:PLATFORM -eq "x64") -and ($XPLAT) -and ($env:PYTHON_VERSION -eq "3.5")) { - # Run C# Performance tests - Write-Host ("Starting performance tests") -ForegroundColor "Green" - if ($XPLAT) { - $CS_PERF_TESTS = ".\src\perf_tests\bin\net461\Python.PerformanceTests.dll" - } - else { - $CS_PERF_TESTS = ".\src\perf_tests\bin\Python.PerformanceTests.dll" - } - &"$CS_RUNNER" "$CS_PERF_TESTS" - $CS_PERF_STATUS = $LastExitCode - if ($CS_PERF_STATUS -ne 0) { - Write-Host "Performance tests (C#) failed" -ForegroundColor "Red" - ReportTime "" - } else { - ReportTime "Performance tests (C#) completed" - } - } else { - Write-Host ("Skipping performance tests for ", $env:PYTHON_VERSION) -ForegroundColor "Yellow" - Write-Host ("on platform ", $env:PLATFORM, " xplat: ", $XPLAT) -ForegroundColor "Yellow" - $CS_PERF_STATUS = 0 - } -} - -if ($XPLAT){ - if ($env:PLATFORM -eq "x64") { - $DOTNET_CMD = "dotnet" - } - else{ - $DOTNET_CMD = "c:\Program Files (x86)\dotnet\dotnet" - } - - # Run Embedded tests for netcoreapp3.1 (OpenCover currently does not supports dotnet core) - Write-Host ("Starting embedded tests for netcoreapp3.1") -ForegroundColor "Green" - &$DOTNET_CMD ".\src\embed_tests\bin\netcoreapp3.1_publish\Python.EmbeddingTest.dll" - $CS_STATUS = $LastExitCode - if ($CS_STATUS -ne 0) { - Write-Host "Embedded tests for netcoreapp3.1 failed" -ForegroundColor "Red" - ReportTime "" - } else { - ReportTime ".NET Core 2.0 tests completed" - } -} - -Write-Host "Timings:" ($timings | Format-Table | Out-String) -ForegroundColor "Green" - -# Set exit code to fail if either Python or Embedded tests failed -if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0 -or $CS_PERF_STATUS -ne 0) { - Write-Host "Tests failed" -ForegroundColor "Red" - $host.SetShouldExit(1) -}