pythonnet · lostmsu · Jan 28, 2021 · Nov 30, 2020 · Dec 2, 2020 · Dec 2, 2020
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -12,9 +12,33 @@ jobs:
       fail-fast: false
       matrix:
         os: [windows, ubuntu, macos]
-        python: [3.6, 3.7, 3.8, 3.9]
+        pyver_minor: [6, 7, 8, 9]
         platform: [x64]
         shutdown_mode: [Normal, Soft]
+        include:
+        - os: ubuntu
+          pyver_minor: 6
+          dll_suffix: m
+        - os: ubuntu
+          pyver_minor: 7
+          dll_suffix: m
+
+        - os: macos
+          dll_prefix: lib
+          dll_pyver_major: '3.'
+          dll_suffix: m
+        - os: ubuntu
+          dll_prefix: lib
+          dll_pyver_major: '3.'
+        - os: windows
+          dll_pyver_major: '3'
+
+        - os: ubuntu
+          dll_ext: .so
+        - os: windows
+          dll_ext: .dll
+        - os: macos
+          dll_ext: .dylib
 
     env:
       PYTHONNET_SHUTDOWN_MODE: ${{ matrix.SHUTDOWN_MODE }}
@@ -32,10 +56,10 @@ jobs:
       - name: Setup .NET
         uses: actions/setup-dotnet@v1
 
-      - name: Set up Python ${{ matrix.python }}
+      - name: Set up Python 3.${{ matrix.pyver_minor }}
         uses: actions/setup-python@v2
         with:
-          python-version: ${{ matrix.python }}
+          python-version: 3.${{ matrix.pyver_minor }}
           architecture: ${{ matrix.platform }}
 
       - name: Install dependencies
@@ -47,16 +71,28 @@ jobs:
           python setup.py configure
           pip install -v .
 
+      # TODO this should be gone once clr module sets PythonDLL or preloads it
+      - name: Python Tests
+        run: pytest
+        if: ${{ matrix.os != 'macos' }}
+        env:
+          PYTHONNET_PYDLL: ${{ matrix.DLL_PREFIX }}python${{matrix.DLL_PYVER_MAJOR}}${{matrix.PYVER_MINOR}}${{matrix.DLL_SUFFIX}}${{matrix.DLL_EXT}}
+
       - name: Python Tests
         run: pytest
+        if: ${{ matrix.os == 'macos' }}
 
       - name: Embedding tests
         run: dotnet test --runtime any-${{ matrix.platform }} src/embed_tests/
         if: ${{ matrix.os != 'macos' }} # Not working right now, doesn't find libpython
+        env:
+          PYTHONNET_PYDLL: ${{ matrix.DLL_PREFIX }}python${{matrix.DLL_PYVER_MAJOR}}${{matrix.PYVER_MINOR}}${{matrix.DLL_SUFFIX}}${{matrix.DLL_EXT}}
 
       - name: Python tests run from .NET
         run: dotnet test --runtime any-${{ matrix.platform }} src/python_tests_runner/
         if: ${{ matrix.os == 'windows' }} # Not working for others right now
+        env:
+          PYTHONNET_PYDLL: ${{ matrix.DLL_PREFIX }}python${{matrix.DLL_PYVER_MAJOR}}${{matrix.PYVER_MINOR}}${{matrix.DLL_SUFFIX}}${{matrix.DLL_EXT}}
 
       # TODO: Run perf tests
       # TODO: Run mono tests on Windows?
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -34,6 +34,8 @@ details about the cause of the failure
 when .NET expects an integer [#1342][i1342]
 -   More specific error messages for method argument mismatch
 -   BREAKING: Methods with `ref` or `out` parameters and void return type return a tuple of only the `ref` and `out` parameters.
+-   BREAKING: to call Python from .NET `Runtime.PythonDLL` property must be set to Python DLL name
+or the DLL must be loaded in advance. This must be done before calling any other Python.NET functions.
 
 ### Fixed
 
@@ -57,6 +59,8 @@ when .NET expects an integer [#1342][i1342]
 ### Removed
 
 -   implicit assembly loading (you have to explicitly `clr.AddReference` before doing import)
+-   support for .NET Framework 4.0-4.6; Mono before 5.4. Python.NET now requires .NET Standard 2.0
+(see [the matrix](https://docs.microsoft.com/en-us/dotnet/standard/net-standard#net-implementation-support))
 
 ## [2.5.0][] - 2020-06-14
 

diff --git a/Directory.Build.props b/Directory.Build.props
@@ -8,7 +8,11 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.CSharp" Version="4.7.0" />
-    <PackageReference Include="NonCopyableAnalyzer" Version="0.6.0">
+    <PackageReference Include="Microsoft.Net.Compilers.Toolset" Version="3.9.0-3.final">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Lost.NonCopyableAnalyzer" Version="0.7.0-m04">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>

diff --git a/appveyor.yml b/appveyor.yml
@@ -30,6 +30,7 @@ environment:
 init:
   # Update Environment Variables based on matrix/platform
   - set PY_VER=%PYTHON_VERSION:.=%
+  - set PYTHONNET_PYDLL=python%PY_VER%.dll
   - set PYTHON=C:\PYTHON%PY_VER%
   - if %PLATFORM%==x64 (set PYTHON=%PYTHON%-x64)
 

diff --git a/src/console/pythonconsole.cs b/src/console/pythonconsole.cs
@@ -26,7 +26,7 @@ private PythonConsole()
         [STAThread]
         public static int Main(string[] args)
         {
-            // Only net40 is capable to safely inject python.runtime.dll into resources.
+            // Only .NET Framework is capable to safely inject python.runtime.dll into resources.
 #if NET40
             // reference the static assemblyLoader to stop it being optimized away
             AssemblyLoader a = assemblyLoader;

diff --git a/src/embed_tests/References.cs b/src/embed_tests/References.cs
@@ -23,7 +23,7 @@ public void Dispose()
         public void MoveToPyObject_SetsNull()
         {
             var dict = new PyDict();
-            NewReference reference = Runtime.PyDict_Items(dict.Handle);
+            NewReference reference = Runtime.PyDict_Items(dict.Reference);
             try
             {
                 Assert.IsFalse(reference.IsNull());
@@ -41,7 +41,7 @@ public void MoveToPyObject_SetsNull()
         public void CanBorrowFromNewReference()
         {
             var dict = new PyDict();
-            NewReference reference = Runtime.PyDict_Items(dict.Handle);
+            NewReference reference = Runtime.PyDict_Items(dict.Reference);
             try
             {
                 PythonException.ThrowIfIsNotZero(Runtime.PyList_Reverse(reference));

diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs
@@ -332,7 +332,7 @@ static void RunAssemblyAndUnload(string domainName)
             // assembly (and Python .NET) to reside
             var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
 
-            theProxy.Call("InitPython", ShutdownMode.Soft);
+            theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Soft, PyRuntime.PythonDLL);
             // From now on use the Proxy to call into the new assembly
             theProxy.RunPython();
 
@@ -400,7 +400,7 @@ static void RunDomainReloadSteps<T1, T2>() where T1 : CrossCaller where T2 : Cro
                 try
                 {
                     var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
-                    theProxy.Call("InitPython", ShutdownMode.Reload);
+                    theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL);
 
                     var caller = CreateInstanceInstanceAndUnwrap<T1>(domain);
                     arg = caller.Execute(arg);
@@ -418,7 +418,7 @@ static void RunDomainReloadSteps<T1, T2>() where T1 : CrossCaller where T2 : Cro
                 try
                 {
                     var theProxy = CreateInstanceInstanceAndUnwrap<Proxy>(domain);
-                    theProxy.Call("InitPython", ShutdownMode.Reload);
+                    theProxy.Call(nameof(PythonRunner.InitPython), ShutdownMode.Reload, PyRuntime.PythonDLL);
 
                     var caller = CreateInstanceInstanceAndUnwrap<T2>(domain);
                     caller.Execute(arg);
@@ -478,8 +478,9 @@ public static void RunPython()
 
         private static IntPtr _state;
 
-        public static void InitPython(ShutdownMode mode)
+        public static void InitPython(ShutdownMode mode, string dllName)
         {
+            PyRuntime.PythonDLL = dllName;
             PythonEngine.Initialize(mode: mode);
             _state = PythonEngine.BeginAllowThreads();
         }

diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs
@@ -96,13 +96,15 @@ public static void PyCheck_Iter_PyObject_IsIterable_ThreadingLock_Test()
             // TypeFlags.HaveIter set in Python 2. This tests a different code path in PyObject_IsIterable and PyIter_Check.
             var threading = Runtime.Runtime.PyImport_ImportModule("threading");
             Exceptions.ErrorCheck(threading);
-            var threadingDict = Runtime.Runtime.PyModule_GetDict(threading);
+            var threadingDict = Runtime.Runtime.PyModule_GetDict(new BorrowedReference(threading));
             Exceptions.ErrorCheck(threadingDict);
             var lockType = Runtime.Runtime.PyDict_GetItemString(threadingDict, "Lock");
-            if (lockType == IntPtr.Zero)
+            if (lockType.IsNull)
                 throw new KeyNotFoundException("class 'Lock' was not found in 'threading'");
 
-            var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType, Runtime.Runtime.PyTuple_New(0));
+            var args = Runtime.Runtime.PyTuple_New(0);
+            var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType.DangerousGetAddress(), args);
+            Runtime.Runtime.XDecref(args);
             Exceptions.ErrorCheck(lockInstance);
 
             Assert.IsFalse(Runtime.Runtime.PyObject_IsIterable(lockInstance));

diff --git a/src/embed_tests/TestTypeManager.cs b/src/embed_tests/TestTypeManager.cs
diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs
@@ -10,10 +10,11 @@ readonly ref struct BorrowedReference
         readonly IntPtr pointer;
         public bool IsNull => this.pointer == IntPtr.Zero;
 
-
         /// <summary>Gets a raw pointer to the Python object</summary>
         public IntPtr DangerousGetAddress()
             => this.IsNull ? throw new NullReferenceException() : this.pointer;
+        /// <summary>Gets a raw pointer to the Python object</summary>
+        public IntPtr DangerousGetAddressOrNull() => this.pointer;
 
         /// <summary>
         /// Creates new instance of <see cref="BorrowedReference"/> from raw pointer. Unsafe.

diff --git a/src/runtime/CustomMarshaler.cs b/src/runtime/CustomMarshaler.cs
@@ -41,8 +41,9 @@ public int GetNativeDataSize()
     /// </summary>
     internal class UcsMarshaler : MarshalerBase
     {
+        internal static readonly int _UCS = Runtime.PyUnicode_GetMax() <= 0xFFFF ? 2 : 4;
+        internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32;
         private static readonly MarshalerBase Instance = new UcsMarshaler();
-        private static readonly Encoding PyEncoding = Runtime.PyEncoding;
 
         public override IntPtr MarshalManagedToNative(object managedObj)
         {
@@ -91,13 +92,13 @@ public static int GetUnicodeByteLength(IntPtr p)
             var len = 0;
             while (true)
             {
-                int c = Runtime._UCS == 2
+                int c = _UCS == 2
                     ? Marshal.ReadInt16(p, len * 2)
                     : Marshal.ReadInt32(p, len * 4);
 
                 if (c == 0)
                 {
-                    return len * Runtime._UCS;
+                    return len * _UCS;
                 }
                 checked
                 {
@@ -147,7 +148,7 @@ public static string PtrToPy3UnicodePy2String(IntPtr p)
     internal class StrArrayMarshaler : MarshalerBase
     {
         private static readonly MarshalerBase Instance = new StrArrayMarshaler();
-        private static readonly Encoding PyEncoding = Runtime.PyEncoding;
+        private static readonly Encoding PyEncoding = UcsMarshaler.PyEncoding;
 
         public override IntPtr MarshalManagedToNative(object managedObj)
         {
@@ -159,7 +160,7 @@ public override IntPtr MarshalManagedToNative(object managedObj)
             }
 
             int totalStrLength = argv.Sum(arg => arg.Length + 1);
-            int memSize = argv.Length * IntPtr.Size + totalStrLength * Runtime._UCS;
+            int memSize = argv.Length * IntPtr.Size + totalStrLength * UcsMarshaler._UCS;
 
             IntPtr mem = Marshal.AllocHGlobal(memSize);
             try

diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs
@@ -11,6 +11,16 @@ ref struct NewReference
     {
         IntPtr pointer;
 
+        /// <summary>Creates a <see cref="NewReference"/> pointing to the same object</summary>
+        public NewReference(BorrowedReference reference, bool canBeNull = false)
+        {
+            var address = canBeNull
+                ? reference.DangerousGetAddressOrNull()
+                : reference.DangerousGetAddress();
+            Runtime.XIncref(address);
+            this.pointer = address;
+        }
+
         [Pure]
         public static implicit operator BorrowedReference(in NewReference reference)
             => new BorrowedReference(reference.pointer);
@@ -28,6 +38,16 @@ public PyObject MoveToPyObject()
             return result;
         }
 
+        /// <summary>Moves ownership of this instance to unmanged pointer</summary>
+        public IntPtr DangerousMoveToPointer()
+        {
+            if (this.IsNull()) throw new NullReferenceException();
+
+            var result = this.pointer;
+            this.pointer = IntPtr.Zero;
+            return result;
+        }
+
         /// <summary>Moves ownership of this instance to unmanged pointer</summary>
         public IntPtr DangerousMoveToPointerOrNull()
         {

diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj
@@ -2,9 +2,9 @@
   <PropertyGroup>
     <TargetFrameworks>netstandard2.0</TargetFrameworks>
     <Platforms>AnyCPU</Platforms>
+    <LangVersion>9.0</LangVersion>
     <RootNamespace>Python.Runtime</RootNamespace>
     <AssemblyName>Python.Runtime</AssemblyName>
-    <LangVersion>9.0</LangVersion>
     <PackageId>pythonnet</PackageId>
     <PackageLicenseUrl>https://github.com/pythonnet/pythonnet/blob/master/LICENSE</PackageLicenseUrl>
     <RepositoryUrl>https://github.com/pythonnet/pythonnet</RepositoryUrl>