Skip to content

Commit 9e5887c

Browse files
lostmsulostmsu
authored
Merge pull request #1365 from losttech/features/VersionIndependent
Build single Python.Runtime.dll for all platforms
2 parents 063a674 + a6cbe20 commit 9e5887c

37 files changed

+1682
-1408
lines changed

‎.github/workflows/main.yml

+39-3
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,33 @@ jobs:
1212
fail-fast: false
1313
matrix:
1414
os: [windows, ubuntu, macos]
15-
python: [3.6, 3.7, 3.8, 3.9]
15+
pyver_minor: [6, 7, 8, 9]
1616
platform: [x64]
1717
shutdown_mode: [Normal, Soft]
18+
include:
19+
- os: ubuntu
20+
pyver_minor: 6
21+
dll_suffix: m
22+
- os: ubuntu
23+
pyver_minor: 7
24+
dll_suffix: m
25+
26+
- os: macos
27+
dll_prefix: lib
28+
dll_pyver_major: '3.'
29+
dll_suffix: m
30+
- os: ubuntu
31+
dll_prefix: lib
32+
dll_pyver_major: '3.'
33+
- os: windows
34+
dll_pyver_major: '3'
35+
36+
- os: ubuntu
37+
dll_ext: .so
38+
- os: windows
39+
dll_ext: .dll
40+
- os: macos
41+
dll_ext: .dylib
1842

1943
env:
2044
PYTHONNET_SHUTDOWN_MODE: ${{ matrix.SHUTDOWN_MODE }}
@@ -32,10 +56,10 @@ jobs:
3256
- name: Setup .NET
3357
uses: actions/setup-dotnet@v1
3458

35-
- name: Set up Python ${{ matrix.python }}
59+
- name: Set up Python 3.${{ matrix.pyver_minor }}
3660
uses: actions/setup-python@v2
3761
with:
38-
python-version: ${{ matrix.python }}
62+
python-version: 3.${{ matrix.pyver_minor }}
3963
architecture: ${{ matrix.platform }}
4064

4165
- name: Install dependencies
@@ -47,16 +71,28 @@ jobs:
4771
python setup.py configure
4872
pip install -v .
4973
74+
# TODO this should be gone once clr module sets PythonDLL or preloads it
75+
- name: Python Tests
76+
run: pytest
77+
if: ${{ matrix.os != 'macos' }}
78+
env:
79+
PYTHONNET_PYDLL: ${{ matrix.DLL_PREFIX }}python${{matrix.DLL_PYVER_MAJOR}}${{matrix.PYVER_MINOR}}${{matrix.DLL_SUFFIX}}${{matrix.DLL_EXT}}
80+
5081
- name: Python Tests
5182
run: pytest
83+
if: ${{ matrix.os == 'macos' }}
5284

5385
- name: Embedding tests
5486
run: dotnet test --runtime any-${{ matrix.platform }} src/embed_tests/
5587
if: ${{ matrix.os != 'macos' }} # Not working right now, doesn't find libpython
88+
env:
89+
PYTHONNET_PYDLL: ${{ matrix.DLL_PREFIX }}python${{matrix.DLL_PYVER_MAJOR}}${{matrix.PYVER_MINOR}}${{matrix.DLL_SUFFIX}}${{matrix.DLL_EXT}}
5690

5791
- name: Python tests run from .NET
5892
run: dotnet test --runtime any-${{ matrix.platform }} src/python_tests_runner/
5993
if: ${{ matrix.os == 'windows' }} # Not working for others right now
94+
env:
95+
PYTHONNET_PYDLL: ${{ matrix.DLL_PREFIX }}python${{matrix.DLL_PYVER_MAJOR}}${{matrix.PYVER_MINOR}}${{matrix.DLL_SUFFIX}}${{matrix.DLL_EXT}}
6096

6197
# TODO: Run perf tests
6298
# TODO: Run mono tests on Windows?

‎CHANGELOG.md

+4
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ details about the cause of the failure
3434
when .NET expects an integer [#1342][i1342]
3535
- More specific error messages for method argument mismatch
3636
- BREAKING: Methods with `ref` or `out` parameters and void return type return a tuple of only the `ref` and `out` parameters.
37+
- BREAKING: to call Python from .NET `Runtime.PythonDLL` property must be set to Python DLL name
38+
or the DLL must be loaded in advance. This must be done before calling any other Python.NET functions.
3739

3840
### Fixed
3941

@@ -57,6 +59,8 @@ when .NET expects an integer [#1342][i1342]
5759
### Removed
5860

5961
- implicit assembly loading (you have to explicitly `clr.AddReference` before doing import)
62+
- support for .NET Framework 4.0-4.6; Mono before 5.4. Python.NET now requires .NET Standard 2.0
63+
(see [the matrix](https://docs.microsoft.com/en-us/dotnet/standard/net-standard#net-implementation-support))
6064

6165
## [2.5.0][] - 2020-06-14
6266

‎Directory.Build.props

+5-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,11 @@
88
</PropertyGroup>
99
<ItemGroup>
1010
<PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
11-
<PackageReference Include="NonCopyableAnalyzer" Version="0.6.0">
11+
<PackageReference Include="Microsoft.Net.Compilers.Toolset" Version="3.9.0-3.final">
12+
<PrivateAssets>all</PrivateAssets>
13+
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
14+
</PackageReference>
15+
<PackageReference Include="Lost.NonCopyableAnalyzer" Version="0.7.0-m04">
1216
<PrivateAssets>all</PrivateAssets>
1317
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
1418
</PackageReference>

‎appveyor.yml

+1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ environment:
3030
init:
3131
# Update Environment Variables based on matrix/platform
3232
- set PY_VER=%PYTHON_VERSION:.=%
33+
- set PYTHONNET_PYDLL=python%PY_VER%.dll
3334
- set PYTHON=C:\PYTHON%PY_VER%
3435
- if %PLATFORM%==x64 (set PYTHON=%PYTHON%-x64)
3536

‎src/console/pythonconsole.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ private PythonConsole()
2626
[STAThread]
2727
public static int Main(string[] args)
2828
{
29-
// Only net40 is capable to safely inject python.runtime.dll into resources.
29+
// Only .NET Framework is capable to safely inject python.runtime.dll into resources.
3030
#if NET40
3131
// reference the static assemblyLoader to stop it being optimized away
3232
AssemblyLoader a = assemblyLoader;

‎src/embed_tests/References.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ public void Dispose()
2323
public void MoveToPyObject_SetsNull()
2424
{
2525
var dict = new PyDict();
26-
NewReference reference = Runtime.PyDict_Items(dict.Handle);
26+
NewReference reference = Runtime.PyDict_Items(dict.Reference);
2727
try
2828
{
2929
Assert.IsFalse(reference.IsNull());
@@ -41,7 +41,7 @@ public void MoveToPyObject_SetsNull()
4141
public void CanBorrowFromNewReference()
4242
{
4343
var dict = new PyDict();
44-
NewReference reference = Runtime.PyDict_Items(dict.Handle);
44+
NewReference reference = Runtime.PyDict_Items(dict.Reference);
4545
try
4646
{
4747
PythonException.ThrowIfIsNotZero(Runtime.PyList_Reverse(reference));

‎src/embed_tests/TestDomainReload.cs

+5-4
Original file line numberDiff line numberDiff line change
@@ -332,7 +332,7 @@ static void RunAssemblyAndUnload(string domainName)
332332
// assembly (and Python .NET) to reside
333333
var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
334334

335-
theProxy.Call("InitPython", ShutdownMode.Soft);
335+
theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Soft, PyRuntime.PythonDLL);
336336
// From now on use the Proxy to call into the new assembly
337337
theProxy.RunPython();
338338

@@ -400,7 +400,7 @@ static void RunDomainReloadSteps<T1, T2>() where T1 : CrossCaller where T2 : Cro
400400
try
401401
{
402402
var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
403-
theProxy.Call("InitPython", ShutdownMode.Reload);
403+
theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL);
404404

405405
var caller = CreateInstanceInstanceAndUnwrap<T1>(domain);
406406
arg = caller.Execute(arg);
@@ -418,7 +418,7 @@ static void RunDomainReloadSteps<T1, T2>() where T1 : CrossCaller where T2 : Cro
418418
try
419419
{
420420
var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
421-
theProxy.Call("InitPython", ShutdownMode.Reload);
421+
theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL);
422422

423423
var caller = CreateInstanceInstanceAndUnwrap<T2>(domain);
424424
caller.Execute(arg);
@@ -478,8 +478,9 @@ public static void RunPython()
478478

479479
private static IntPtr _state;
480480

481-
public static void InitPython(ShutdownMode mode)
481+
public static void InitPython(ShutdownMode mode, string dllName)
482482
{
483+
PyRuntime.PythonDLL = dllName;
483484
PythonEngine.Initialize(mode: mode);
484485
_state = PythonEngine.BeginAllowThreads();
485486
}

‎src/embed_tests/TestRuntime.cs

+5-3
Original file line numberDiff line numberDiff line change
@@ -96,13 +96,15 @@ public static void PyCheck_Iter_PyObject_IsIterable_ThreadingLock_Test()
9696
// TypeFlags.HaveIter set in Python 2. This tests a different code path in PyObject_IsIterable and PyIter_Check.
9797
var threading = Runtime.Runtime.PyImport_ImportModule("threading");
9898
Exceptions.ErrorCheck(threading);
99-
var threadingDict = Runtime.Runtime.PyModule_GetDict(threading);
99+
var threadingDict = Runtime.Runtime.PyModule_GetDict(new BorrowedReference(threading));
100100
Exceptions.ErrorCheck(threadingDict);
101101
var lockType = Runtime.Runtime.PyDict_GetItemString(threadingDict, "Lock");
102-
if (lockType == IntPtr.Zero)
102+
if (lockType.IsNull)
103103
throw new KeyNotFoundException("class 'Lock' was not found in 'threading'");
104104

105-
var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType, Runtime.Runtime.PyTuple_New(0));
105+
var args = Runtime.Runtime.PyTuple_New(0);
106+
var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType.DangerousGetAddress(), args);
107+
Runtime.Runtime.XDecref(args);
106108
Exceptions.ErrorCheck(lockInstance);
107109

108110
Assert.IsFalse(Runtime.Runtime.PyObject_IsIterable(lockInstance));

‎src/embed_tests/TestTypeManager.cs

-65
This file was deleted.

‎src/runtime/BorrowedReference.cs

+2-1
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,11 @@ readonly ref struct BorrowedReference
1010
readonly IntPtr pointer;
1111
public bool IsNull => this.pointer == IntPtr.Zero;
1212

13-
1413
/// <summary>Gets a raw pointer to the Python object</summary>
1514
public IntPtr DangerousGetAddress()
1615
=> this.IsNull ? throw new NullReferenceException() : this.pointer;
16+
/// <summary>Gets a raw pointer to the Python object</summary>
17+
public IntPtr DangerousGetAddressOrNull() => this.pointer;
1718

1819
/// <summary>
1920
/// Creates new instance of <see cref="BorrowedReference"/> from raw pointer. Unsafe.

‎src/runtime/CustomMarshaler.cs

+6-5
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,9 @@ public int GetNativeDataSize()
4141
/// </summary>
4242
internal class UcsMarshaler : MarshalerBase
4343
{
44+
internal static readonly int _UCS = Runtime.PyUnicode_GetMax() <= 0xFFFF ? 2 : 4;
45+
internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32;
4446
private static readonly MarshalerBase Instance = new UcsMarshaler();
45-
private static readonly Encoding PyEncoding = Runtime.PyEncoding;
4647

4748
public override IntPtr MarshalManagedToNative(object managedObj)
4849
{
@@ -91,13 +92,13 @@ public static int GetUnicodeByteLength(IntPtr p)
9192
var len = 0;
9293
while (true)
9394
{
94-
int c = Runtime._UCS == 2
95+
int c = _UCS == 2
9596
? Marshal.ReadInt16(p, len * 2)
9697
: Marshal.ReadInt32(p, len * 4);
9798

9899
if (c == 0)
99100
{
100-
return len * Runtime._UCS;
101+
return len * _UCS;
101102
}
102103
checked
103104
{
@@ -147,7 +148,7 @@ public static string PtrToPy3UnicodePy2String(IntPtr p)
147148
internal class StrArrayMarshaler : MarshalerBase
148149
{
149150
private static readonly MarshalerBase Instance = new StrArrayMarshaler();
150-
private static readonly Encoding PyEncoding = Runtime.PyEncoding;
151+
private static readonly Encoding PyEncoding = UcsMarshaler.PyEncoding;
151152

152153
public override IntPtr MarshalManagedToNative(object managedObj)
153154
{
@@ -159,7 +160,7 @@ public override IntPtr MarshalManagedToNative(object managedObj)
159160
}
160161

161162
int totalStrLength = argv.Sum(arg => arg.Length + 1);
162-
int memSize = argv.Length * IntPtr.Size + totalStrLength * Runtime._UCS;
163+
int memSize = argv.Length * IntPtr.Size + totalStrLength * UcsMarshaler._UCS;
163164

164165
IntPtr mem = Marshal.AllocHGlobal(memSize);
165166
try

‎src/runtime/NewReference.cs

+20
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,16 @@ ref struct NewReference
1111
{
1212
IntPtr pointer;
1313

14+
/// <summary>Creates a <see cref="NewReference"/> pointing to the same object</summary>
15+
public NewReference(BorrowedReference reference, bool canBeNull = false)
16+
{
17+
var address = canBeNull
18+
? reference.DangerousGetAddressOrNull()
19+
: reference.DangerousGetAddress();
20+
Runtime.XIncref(address);
21+
this.pointer = address;
22+
}
23+
1424
[Pure]
1525
public static implicit operator BorrowedReference(in NewReference reference)
1626
=> new BorrowedReference(reference.pointer);
@@ -28,6 +38,16 @@ public PyObject MoveToPyObject()
2838
return result;
2939
}
3040

41+
/// <summary>Moves ownership of this instance to unmanged pointer</summary>
42+
public IntPtr DangerousMoveToPointer()
43+
{
44+
if (this.IsNull()) throw new NullReferenceException();
45+
46+
var result = this.pointer;
47+
this.pointer = IntPtr.Zero;
48+
return result;
49+
}
50+
3151
/// <summary>Moves ownership of this instance to unmanged pointer</summary>
3252
public IntPtr DangerousMoveToPointerOrNull()
3353
{

‎src/runtime/Python.Runtime.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22
<PropertyGroup>
33
<TargetFrameworks>netstandard2.0</TargetFrameworks>
44
<Platforms>AnyCPU</Platforms>
5+
<LangVersion>9.0</LangVersion>
56
<RootNamespace>Python.Runtime</RootNamespace>
67
<AssemblyName>Python.Runtime</AssemblyName>
7-
<LangVersion>9.0</LangVersion>
88
<PackageId>pythonnet</PackageId>
99
<PackageLicenseUrl>https://github.com/pythonnet/pythonnet/blob/master/LICENSE</PackageLicenseUrl>
1010
<RepositoryUrl>https://github.com/pythonnet/pythonnet</RepositoryUrl>

0 commit comments

Comments
 (0)