Merge branch 'master' into losttech-perf-interop

filmor · web-flow · commit 34c2744b6c9f · 2020-02-26T12:23:03.000+01:00
diff --git a/.travis.yml b/.travis.yml
@@ -44,7 +44,7 @@ install:
 
 script:
   - python -m pytest
-  - $RUN_TESTS src/embed_tests/bin/$EMBED_TESTS_PATH/Python.EmbeddingTest.dll
+  - $RUN_TESTS src/embed_tests/bin/$EMBED_TESTS_PATH/Python.EmbeddingTest.dll --labels=All
   # does not work on Linux, because NuGet package for 2.3 is Windows only
   # - "if [[ $TRAVIS_PYTHON_VERSION == '3.5' && $PERF_TESTS_PATH != '' ]]; then mono $NUNIT_PATH src/perf_tests/bin/$PERF_TESTS_PATH/Python.PerformanceTests.dll; fi"
 
diff --git a/AUTHORS.md b/AUTHORS.md
@@ -15,6 +15,7 @@
 -   Alex Earl ([@slide](https://github.com/slide))
 -   Alex Helms ([@alexhelms](https://github.com/alexhelms))
 -   Alexandre Catarino([@AlexCatarino](https://github.com/AlexCatarino))
+-   Andrey Sant'Anna ([@andreydani](https://github.com/andreydani))
 -   Arvid JB ([@ArvidJB](https://github.com/ArvidJB))
 -   Benoît Hudson ([@benoithudson](https://github.com/benoithudson))
 -   Bradley Friedman ([@leith-bartrich](https://github.com/leith-bartrich))
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -30,6 +30,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][].
   together with Nuitka
 - Fixes bug where delegates get casts (dotnetcore)
 - Determine size of interpreter longs at runtime
+- Handling exceptions ocurred in ModuleObject's getattribute 
 
 ## [2.4.0][]
 
diff --git a/ci/appveyor_run_tests.ps1 b/ci/appveyor_run_tests.ps1
@@ -55,7 +55,7 @@ if ($PYTHON_STATUS -ne 0) {
 # Run Embedded tests with C# coverage
 Write-Host ("Starting embedded tests") -ForegroundColor "Green"
 .$OPENCOVER -register:user -searchdirs:"$RUNTIME_DIR" -output:cs.coverage `
-            -target:"$CS_RUNNER" -targetargs:"$CS_TESTS" `
+            -target:"$CS_RUNNER" -targetargs:"$CS_TESTS --labels=All" `
             -filter:"+[*]Python.Runtime*" `
             -returntargetcode
 $CS_STATUS = $LastExitCode
diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj
@@ -77,13 +77,16 @@
 
   
   <ItemGroup>
-    <PackageReference Include="NUnit" Version="3.7.1" />
-    <PackageReference Include="NUnit.ConsoleRunner" Version="3.7.0" />
-    <PackageReference Include="NUnit3TestAdapter" Version="3.8.0" />
-    <PackageReference Include="NUnitLite" Version="3.7.2" />
+    <PackageReference Include="NUnit" Version="3.12.0" />
+    <PackageReference Include="NUnit.ConsoleRunner" Version="3.11.1" />
+    <PackageReference Include="NUnit3TestAdapter" Version="3.16.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="NUnitLite" Version="3.12.0" />
   </ItemGroup>
   <ItemGroup Condition="'$(TargetFramework)' == 'netcoreapp2.0'">
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.0.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\runtime\Python.Runtime.15.csproj" />
diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj
@@ -70,8 +70,8 @@
   </PropertyGroup>
   <ItemGroup>
     <Reference Include="Microsoft.CSharp" />
-    <Reference Include="nunit.framework, Version=3.7.1.0, Culture=neutral, PublicKeyToken=2638cd05610744eb, processorArchitecture=MSIL">
-      <HintPath>..\..\packages\NUnit.3.7.1\lib\net40\nunit.framework.dll</HintPath>
+    <Reference Include="nunit.framework, Version=3.12.0.0, Culture=neutral, PublicKeyToken=2638cd05610744eb, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\NUnit.3.12.0\lib\net40\nunit.framework.dll</HintPath>
     </Reference>
     <Reference Include="System" />
   </ItemGroup>
@@ -126,4 +126,10 @@
     <Copy SourceFiles="$(TargetAssembly)" DestinationFolder="$(PythonBuildDir)" />
     <!--Copy SourceFiles="$(TargetAssemblyPdb)" Condition="Exists('$(TargetAssemblyPdb)')" DestinationFolder="$(PythonBuildDir)" /-->
   </Target>
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('..\..\packages\NUnit.3.12.0\build\NUnit.props')" Text="$([System.String]::Format('$(ErrorText)', '..\..\packages\NUnit.3.12.0\build\NUnit.props'))" />
+  </Target>
 </Project>
diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using NUnit.Framework;
 using Python.Runtime;
 using Python.Runtime.Platform;
@@ -110,9 +111,15 @@ public static void PyCheck_Iter_PyObject_IsIterable_ThreadingLock_Test()
             // Create an instance of threading.Lock, which is one of the very few types that does not have the
             // TypeFlags.HaveIter set in Python 2. This tests a different code path in PyObject_IsIterable and PyIter_Check.
             var threading = Runtime.Runtime.PyImport_ImportModule("threading");
+            Exceptions.ErrorCheck(threading);
             var threadingDict = Runtime.Runtime.PyModule_GetDict(threading);
+            Exceptions.ErrorCheck(threadingDict);
             var lockType = Runtime.Runtime.PyDict_GetItemString(threadingDict, "Lock");
+            if (lockType == IntPtr.Zero)
+                throw new KeyNotFoundException("class 'Lock' was not found in 'threading'");
+
             var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType, Runtime.Runtime.PyTuple_New(0));
+            Exceptions.ErrorCheck(lockInstance);
 
             Assert.IsFalse(Runtime.Runtime.PyObject_IsIterable(lockInstance));
             Assert.IsFalse(Runtime.Runtime.PyIter_Check(lockInstance));
diff --git a/src/embed_tests/packages.config b/src/embed_tests/packages.config
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="NUnit" version="3.7.1" targetFramework="net40" />
-  <package id="NUnit.ConsoleRunner" version="3.7.0" targetFramework="net40" />
+  <package id="NUnit" version="3.12.0" targetFramework="net40" />
+  <package id="NUnit.ConsoleRunner" version="3.11.1" targetFramework="net40" />
 </packages>
diff --git a/src/perf_tests/BaselineComparisonConfig.cs b/src/perf_tests/BaselineComparisonConfig.cs
@@ -5,6 +5,7 @@
 
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Horology;
 
 namespace Python.PerformanceTests
 {
@@ -18,7 +19,11 @@ public BaselineComparisonConfig()
 
             string deploymentRoot = BenchmarkTests.DeploymentRoot;
 
-            var baseJob = Job.Default;
+            var baseJob = Job.Default
+                .WithLaunchCount(1)
+                .WithWarmupCount(3)
+                .WithMaxIterationCount(100)
+                .WithIterationTime(TimeInterval.FromMilliseconds(100));
             this.Add(baseJob
                 .WithId("baseline")
                 .WithEnvironmentVariable(EnvironmentVariableName,
diff --git a/src/perf_tests/Python.PerformanceTests.csproj b/src/perf_tests/Python.PerformanceTests.csproj
@@ -11,14 +11,17 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="BenchmarkDotNet" Version="0.11.5" />
+    <PackageReference Include="BenchmarkDotNet" Version="0.12.0" />
     <PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.0">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="nunit" Version="3.12.0" />
-    <PackageReference Include="NUnit3TestAdapter" Version="3.13.0" />
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.2.0" />
+    <PackageReference Include="NUnit3TestAdapter" Version="3.16.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.5.0" />
     <PackageReference Include="pythonnet" Version="2.3.0" GeneratePathProperty="true">
       <IncludeAssets>compile</IncludeAssets>
     </PackageReference>
diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs
@@ -0,0 +1,22 @@
+namespace Python.Runtime
+{
+    using System;
+    /// <summary>
+    /// Represents a reference to a Python object, that is being lent, and
+    /// can only be safely used until execution returns to the caller.
+    /// </summary>
+    readonly ref struct BorrowedReference
+    {
+        readonly IntPtr pointer;
+        public bool IsNull => this.pointer == IntPtr.Zero;
+
+        /// <summary>Gets a raw pointer to the Python object</summary>
+        public IntPtr DangerousGetAddress()
+            => this.IsNull ? throw new NullReferenceException() : this.pointer;
+
+        BorrowedReference(IntPtr pointer)
+        {
+            this.pointer = pointer;
+        }
+    }
+}
diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs
@@ -0,0 +1,39 @@
+namespace Python.Runtime
+{
+    using System;
+    /// <summary>
+    /// Represents a reference to a Python object, that is tracked by Python's reference counting.
+    /// </summary>
+    [NonCopyable]
+    ref struct NewReference
+    {
+        IntPtr pointer;
+        public bool IsNull => this.pointer == IntPtr.Zero;
+
+        /// <summary>Gets a raw pointer to the Python object</summary>
+        public IntPtr DangerousGetAddress()
+            => this.IsNull ? throw new NullReferenceException() : this.pointer;
+
+        /// <summary>
+        /// Returns <see cref="PyObject"/> wrapper around this reference, which now owns
+        /// the pointer. Sets the original reference to <c>null</c>, as it no longer owns it.
+        /// </summary>
+        public PyObject MoveToPyObject()
+        {
+            if (this.IsNull) throw new NullReferenceException();
+
+            var result = new PyObject(this.pointer);
+            this.pointer = IntPtr.Zero;
+            return result;
+        }
+        /// <summary>
+        /// Removes this reference to a Python object, and sets it to <c>null</c>.
+        /// </summary>
+        public void Dispose()
+        {
+            if (!this.IsNull)
+                Runtime.XDecref(this.pointer);
+            this.pointer = IntPtr.Zero;
+        }
+    }
+}
diff --git a/src/runtime/NonCopyableAttribute.cs b/src/runtime/NonCopyableAttribute.cs
@@ -0,0 +1,6 @@
+namespace Python.Runtime
+{
+    using  System;
+    [AttributeUsage(AttributeTargets.Struct)]
+    class NonCopyableAttribute : Attribute { }
+}
diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj
@@ -129,6 +129,13 @@
     <PackageReference Include="Microsoft.TargetingPack.NETFramework.v4.5" Version="1.0.1" ExcludeAssets="All" PrivateAssets="All" />
   </ItemGroup>
 
+  <ItemGroup>
+    <PackageReference Include="NonCopyableAnalyzer" Version="0.5.1">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
+  </ItemGroup>
+
   <Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
 
   <PropertyGroup>
diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj
@@ -83,6 +83,7 @@
     </Compile>
     <Compile Include="arrayobject.cs" />
     <Compile Include="assemblymanager.cs" />
+    <Compile Include="BorrowedReference.cs" />
     <Compile Include="classderived.cs" />
     <Compile Include="classbase.cs" />
     <Compile Include="classmanager.cs" />
@@ -119,6 +120,8 @@
     <Compile Include="moduleobject.cs" />
     <Compile Include="modulepropertyobject.cs" />
     <Compile Include="nativecall.cs" />
+    <Compile Include="NewReference.cs" />
+    <Compile Include="NonCopyableAttribute.cs" />
     <Compile Include="overload.cs" />
     <Compile Include="propertyobject.cs" />
     <Compile Include="pyansistring.cs" />
diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs
@@ -3,7 +3,7 @@
 
 namespace Python.Runtime
 {
-    internal class Util
+    internal static class Util
     {
         internal static Int64 ReadCLong(IntPtr tp, int offset)
         {
@@ -29,5 +29,12 @@ internal static void WriteCLong(IntPtr type, int offset, Int64 flags)
                 Marshal.WriteInt64(type, offset, flags);
             }
         }
+
+        /// <summary>
+        /// Null-coalesce: if <paramref name="primary"/> parameter is not
+        /// <see cref="IntPtr.Zero"/>, return it. Otherwise return <paramref name="fallback"/>.
+        /// </summary>
+        internal static IntPtr Coalesce(this IntPtr primary, IntPtr fallback)
+            => primary == IntPtr.Zero ? fallback : primary;
     }
-}
+}
diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs
@@ -145,7 +145,7 @@ internal static void UpdatePath()
                 probed.Clear();
                 for (var i = 0; i < count; i++)
                 {
-                    IntPtr item = Runtime.PyList_GetItem(list, i);
+                    BorrowedReference item = Runtime.PyList_GetItem(list, i);
                     string path = Runtime.GetManagedString(item);
                     if (path != null)
                     {
@@ -492,4 +492,4 @@ internal static Type[] GetTypes(Assembly a)
             }
         }
     }
-}
+}
diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs
@@ -292,7 +292,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth
                 for (int i = 0; i < pynkwargs; ++i)
                 {
                     var keyStr = Runtime.GetManagedString(Runtime.PyList_GetItem(keylist, i));
-                    kwargDict[keyStr] = Runtime.PyList_GetItem(valueList, i);
+                    kwargDict[keyStr] = Runtime.PyList_GetItem(valueList, i).DangerousGetAddress();
                 }
                 Runtime.XDecref(keylist);
                 Runtime.XDecref(valueList);
diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs
@@ -280,7 +280,18 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key)
                 return self.dict;
             }
 
-            ManagedType attr = self.GetAttribute(name, true);
+            ManagedType attr = null;
+
+            try
+            {
+                attr = self.GetAttribute(name, true);
+            }
+            catch (Exception e)
+            {
+                Exceptions.SetError(e);
+                return IntPtr.Zero;
+            }
+            
 
             if (attr == null)
             {
diff --git a/src/runtime/pydict.cs b/src/runtime/pydict.cs
@@ -139,12 +139,20 @@ public PyObject Values()
         /// </remarks>
         public PyObject Items()
         {
-            IntPtr items = Runtime.PyDict_Items(obj);
-            if (items == IntPtr.Zero)
+            var items = Runtime.PyDict_Items(this.obj);
+            try
             {
-                throw new PythonException();
+                if (items.IsNull)
+                {
+                    throw new PythonException();
+                }
+
+                return items.MoveToPyObject();
+            }
+            finally
+            {
+                items.Dispose();
             }
-            return new PyObject(items);
         }
 
 
diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs
@@ -758,11 +758,14 @@ public static void With(PyObject obj, Action<dynamic> Body)
             catch (PythonException e)
             {
                 ex = e;
-                type = ex.PyType;
-                val = ex.PyValue;
-                traceBack = ex.PyTB;
+                type = ex.PyType.Coalesce(type);
+                val = ex.PyValue.Coalesce(val);
+                traceBack = ex.PyTB.Coalesce(traceBack);
             }
 
+            Runtime.XIncref(type);
+            Runtime.XIncref(val);
+            Runtime.XIncref(traceBack);
             var exitResult = obj.InvokeMethod("__exit__", new PyObject(type), new PyObject(val), new PyObject(traceBack));
 
             if (ex != null && !exitResult.IsTrue()) throw ex;
diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs
@@ -21,7 +21,7 @@ public class PythonException : System.Exception, IPyDisposable
         public PythonException()
         {
             IntPtr gs = PythonEngine.AcquireLock();
-            Runtime.PyErr_Fetch(ref _pyType, ref _pyValue, ref _pyTB);
+            Runtime.PyErr_Fetch(out _pyType, out _pyValue, out _pyTB);
             if (_pyType != IntPtr.Zero && _pyValue != IntPtr.Zero)
             {
                 string type;
@@ -161,12 +161,23 @@ public void Dispose()
                 if (Runtime.Py_IsInitialized() > 0 && !Runtime.IsFinalizing)
                 {
                     IntPtr gs = PythonEngine.AcquireLock();
-                    Runtime.XDecref(_pyType);
-                    Runtime.XDecref(_pyValue);
+                    if (_pyType != IntPtr.Zero)
+                    {
+                        Runtime.XDecref(_pyType);
+                        _pyType= IntPtr.Zero;
+                    }
+
+                    if (_pyValue != IntPtr.Zero)
+                    {
+                        Runtime.XDecref(_pyValue);
+                        _pyValue = IntPtr.Zero;
+                    }
+
                     // XXX Do we ever get TraceBack? //
                     if (_pyTB != IntPtr.Zero)
                     {
                         Runtime.XDecref(_pyTB);
+                        _pyTB = IntPtr.Zero;
                     }
                     PythonEngine.ReleaseLock(gs);
                 }
diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs
