diff --git a/.bumpversion.cfg b/.bumpversion.cfg index ac1d31324..01b9ae991 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.4.0.dev0 +current_version = 1.0.5.30 parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? serialize = {major}.{minor}.{patch}.{release}{dev} diff --git a/.travis.yml b/.travis.yml index d059bdcde..900e207a7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,6 @@ matrix: - dotnet-hostfxr-2.0.0 - dotnet-runtime-2.0.0 - dotnet-sdk-2.0.0 - - python: 3.4 env: *xplat-env addons: *xplat-addons @@ -34,24 +33,9 @@ matrix: - python: 3.6 env: *xplat-env addons: *xplat-addons - - - python: 3.7 + - python: "3.7-dev" env: *xplat-env - dist: xenial - sudo: true - addons: &xplat-addons-xenial - apt: - sources: - - sourceline: deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod xenial main - key_url: https://packages.microsoft.com/keys/microsoft.asc - - sourceline: deb https://download.mono-project.com/repo/ubuntu stable-xenial main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF - packages: - - mono-devel - - ca-certificates-mono - - dotnet-hostfxr-2.0.0 - - dotnet-runtime-2.0.0 - - dotnet-sdk-2.0.0 + addons: *xplat-addons # --------------------- Classic builds ------------------------ - python: 2.7 @@ -68,18 +52,15 @@ matrix: - python: 3.6 env: *classic-env - - python: 3.7 + - python: "3.7-dev" + env: *classic-env + + allow_failures: + - python: "3.7-dev" + env: *xplat-env + + - python: "3.7-dev" env: *classic-env - dist: xenial - sudo: true - addons: - apt: - sources: - - sourceline: deb http://download.mono-project.com/repo/ubuntu xenial main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF - packages: - - mono-devel - - ca-certificates-mono env: global: diff --git a/AUTHORS.md b/AUTHORS.md index fe2d2b172..fc0fdf86b 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -14,7 +14,6 @@ - Alexandre Catarino([@AlexCatarino](https://github.com/AlexCatarino)) - Arvid JB ([@ArvidJB](https://github.com/ArvidJB)) -- Benoît Hudson ([@benoithudson](https://github.com/benoithudson)) - Bradley Friedman ([@leith-bartrich](https://github.com/leith-bartrich)) - Callum Noble ([@callumnoble](https://github.com/callumnoble)) - Christian Heimes ([@tiran](https://github.com/tiran)) @@ -23,12 +22,10 @@ - Daniel Fernandez ([@fdanny](https://github.com/fdanny)) - Daniel Santana ([@dgsantana](https://github.com/dgsantana)) - Dave Hirschfeld ([@dhirschfeld](https://github.com/dhirschfeld)) -- David Lassonde ([@lassond](https://github.com/lassond)) - David Lechner ([@dlech](https://github.com/dlech)) - Dmitriy Se ([@dmitriyse](https://github.com/dmitriyse)) - He-chien Tsai ([@t3476](https://github.com/t3476)) -   Ivan Cronyn ([@cronan](https://github.com/cronan)) -- Jan Krivanek ([@jakrivan](https://github.com/jakrivan)) -   Jeff Reback ([@jreback](https://github.com/jreback)) - Joe Frayne ([@jfrayne](https://github.com/jfrayne)) - John Burnett ([@johnburnett](https://github.com/johnburnett)) @@ -42,8 +39,7 @@ - Sam Winstanley ([@swinstanley](https://github.com/swinstanley)) - Sean Freitag ([@cowboygneox](https://github.com/cowboygneox)) - Serge Weinstock ([@sweinst](https://github.com/sweinst)) -- Simon Mourier ([@smourier](https://github.com/smourier)) -- Viktoria Kovescses ([@vkovec](https://github.com/vkovec)) +- Victor Milovanov([@lostmsu](https://github.com/lostmsu)) - Ville M. Vainio ([@vivainio](https://github.com/vivainio)) - Virgil Dupras ([@hsoft](https://github.com/hsoft)) - Wenguang Yang ([@yagweb](https://github.com/yagweb)) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3816cd0c..b92703d43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,35 +8,26 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. ## [unreleased][] ### Added - - Added support for embedding python into dotnet core 2.0 (NetStandard 2.0) - Added new build system (pythonnet.15.sln) based on dotnetcore-sdk/xplat(crossplatform msbuild). Currently there two side-by-side build systems that produces the same output (net40) from the same sources. After a some transition time, current (mono/ msbuild 14.0) build system will be removed. - NUnit upgraded to 3.7 (eliminates travis-ci random bug) -- Added C# `PythonEngine.AddShutdownHandler` to help client code clean up on shutdown. - Added `clr.GetClrType` ([#432][i432])([#433][p433]) - Allowed passing `None` for nullable args ([#460][p460]) - Added keyword arguments based on C# syntax for calling CPython methods ([#461][p461]) - Catches exceptions thrown in C# iterators (yield returns) and rethrows them in python ([#475][i475])([#693][p693]) - Implemented GetDynamicMemberNames() for PyObject to allow dynamic object members to be visible in the debugger ([#443][i443])([#690][p690]) - Incorporated reference-style links to issues and pull requests in the CHANGELOG ([#608][i608]) -- Added detailed comments about aproaches and dangers to handle multi-app-domains ([#625][p625]) -- Python 3.7 support, builds and testing added. Defaults changed from Python 3.6 to 3.7 ([#698][p698]) +- Added PyObject finalizer support, Python objects referred by C# can be auto collect now. ### Changed - -- Reattach python exception traceback information (#545) -- PythonEngine.Intialize will now call `Py_InitializeEx` with a default value of 0, so signals will not be configured by default on embedding. This is different from the previous behaviour, where `Py_Initialize` was called instead, which sets initSigs to 1. ([#449][i449]) +- PythonException included C# call stack +- improved performance of calls from Python to C# ### Fixed -- Fixed secondary PythonEngine.Initialize call, all sensitive static variables now reseted. - This is a hidden bug. Once python cleaning up enough memory, objects from previous engine run becomes corrupted. ([#534][p534]) - Fixed Visual Studio 2017 compat ([#434][i434]) for setup.py -- Fixed crashes when integrating pythonnet in Unity3d ([#714][i714]), - related to unloading the Application Domain -- Fixed interop methods with Py_ssize_t. NetCoreApp 2.0 is more sensitive than net40 and requires this fix. ([#531][p531]) - Fixed crash on exit of the Python interpreter if a python class derived from a .NET class has a `__namespace__` or `__assembly__` attribute ([#481][i481]) @@ -48,8 +39,6 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Fixed errors breaking .NET Remoting on method invoke ([#276][i276]) - Fixed PyObject.GetHashCode ([#676][i676]) - Fix memory leaks due to spurious handle incrementation ([#691][i691]) -- Fix spurious assembly loading exceptions from private types ([#703][i703]) -- Fix inheritance of non-abstract base methods ([#755][i755]) ## [2.3.0][] - 2017-03-11 @@ -607,7 +596,6 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. [1.0.0]: https://github.com/pythonnet/pythonnet/releases/tag/1.0 -[i714]: https://github.com/pythonnet/pythonnet/issues/714 [i608]: https://github.com/pythonnet/pythonnet/issues/608 [i443]: https://github.com/pythonnet/pythonnet/issues/443 [p690]: https://github.com/pythonnet/pythonnet/pull/690 @@ -698,9 +686,4 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. [p225]: https://github.com/pythonnet/pythonnet/pull/225 [p78]: https://github.com/pythonnet/pythonnet/pull/78 [p163]: https://github.com/pythonnet/pythonnet/pull/163 -[p625]: https://github.com/pythonnet/pythonnet/pull/625 [i131]: https://github.com/pythonnet/pythonnet/issues/131 -[p531]: https://github.com/pythonnet/pythonnet/pull/531 -[i755]: https://github.com/pythonnet/pythonnet/pull/755 -[p534]: https://github.com/pythonnet/pythonnet/pull/534 -[i449]: https://github.com/pythonnet/pythonnet/issues/449 diff --git a/NuGet.config b/NuGet.config index 5210cd6c9..719fbc83c 100644 --- a/NuGet.config +++ b/NuGet.config @@ -1,4 +1,4 @@ - + diff --git a/appveyor.yml b/appveyor.yml index ce345cbf8..6bebef490 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,19 +23,10 @@ environment: BUILD_OPTS: --xplat - PYTHON_VERSION: 3.6 BUILD_OPTS: --xplat - - PYTHON_VERSION: 3.7 - BUILD_OPTS: --xplat - PYTHON_VERSION: 2.7 - PYTHON_VERSION: 3.4 - PYTHON_VERSION: 3.5 - PYTHON_VERSION: 3.6 - - PYTHON_VERSION: 3.7 - -matrix: - allow_failures: - - PYTHON_VERSION: 3.4 - BUILD_OPTS: --xplat - - PYTHON_VERSION: 3.4 init: # Update Environment Variables based on matrix/platform diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 3641185bb..e2c407435 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,6 +1,6 @@ package: name: pythonnet - version: "2.4.0.dev0" + version: "1.0.5.30" build: skip: True # [not win] diff --git a/setup.py b/setup.py index 1b6f07ea6..8e72acc96 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ # Allow config/verbosity to be set from cli # http://stackoverflow.com/a/4792601/5208670 CONFIG = "Release" # Release or Debug -VERBOSITY = "normal" # quiet, minimal, normal, detailed, diagnostic +VERBOSITY = "minimal" # quiet, minimal, normal, detailed, diagnostic is_64bits = sys.maxsize > 2**32 DEVTOOLS = "MsDev" if sys.platform == "win32" else "Mono" @@ -329,16 +329,9 @@ def _install_packages(self): self.debug_print("Updating NuGet: {0}".format(cmd)) subprocess.check_call(cmd, shell=use_shell) - try: - # msbuild=14 is mainly for Mono issues - cmd = "{0} restore pythonnet.sln -MSBuildVersion 14 -o packages".format(nuget) - self.debug_print("Installing packages: {0}".format(cmd)) - subprocess.check_call(cmd, shell=use_shell) - except: - # when only VS 2017 is installed do not specify msbuild version - cmd = "{0} restore pythonnet.sln -o packages".format(nuget) - self.debug_print("Installing packages: {0}".format(cmd)) - subprocess.check_call(cmd, shell=use_shell) + cmd = "{0} restore pythonnet.sln -o packages".format(nuget) + self.debug_print("Installing packages: {0}".format(cmd)) + subprocess.check_call(cmd, shell=use_shell) def _find_msbuild_tool(self, tool="msbuild.exe", use_windows_sdk=False): """Return full path to one of the Microsoft build tools""" @@ -492,7 +485,7 @@ def run(self): setup( name="pythonnet", - version="2.4.0.dev0", + version="1.0.5.30", description=".Net and Mono integration for Python", url='https://pythonnet.github.io/', license='MIT', @@ -523,10 +516,10 @@ def run(self): 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', 'Operating System :: Microsoft :: Windows', 'Operating System :: POSIX :: Linux', 'Operating System :: MacOS :: MacOS X', diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index c164e75d6..3d2192f56 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("2.4.0")] +[assembly: AssemblyVersion("1.0.5.30")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 3bb3a533b..62ec1ad42 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("2.4.0"), + Version = new Version("1.0.5.30"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/clrmodule/clrmodule.csproj b/src/clrmodule/clrmodule.csproj index 6e5ff4966..374948d89 100644 --- a/src/clrmodule/clrmodule.csproj +++ b/src/clrmodule/clrmodule.csproj @@ -1,5 +1,5 @@ - - + + Debug AnyCPU @@ -9,7 +9,7 @@ clrmodule bin\clrmodule.xml bin\ - v4.0 + v4.5.2 1591 ..\..\ @@ -92,4 +92,4 @@ - + \ No newline at end of file diff --git a/src/clrmodule/packages.config b/src/clrmodule/packages.config index 2a95dc54d..bfd9f475d 100644 --- a/src/clrmodule/packages.config +++ b/src/clrmodule/packages.config @@ -1,4 +1,4 @@ - + - - + + \ No newline at end of file diff --git a/src/console/Console.csproj b/src/console/Console.csproj index ea88b6356..eb54c9209 100644 --- a/src/console/Console.csproj +++ b/src/console/Console.csproj @@ -1,5 +1,5 @@ - - + + Debug AnyCPU @@ -9,7 +9,7 @@ Python.Runtime bin\nPython.xml bin\ - v4.0 + v4.5.2 1591 ..\..\ @@ -98,4 +98,4 @@ - + \ No newline at end of file diff --git a/src/embed_tests/GlobalTestsSetup.cs b/src/embed_tests/GlobalTestsSetup.cs deleted file mode 100644 index 458ab6a99..000000000 --- a/src/embed_tests/GlobalTestsSetup.cs +++ /dev/null @@ -1,21 +0,0 @@ -using NUnit.Framework; -using Python.Runtime; - -namespace Python.EmbeddingTest -{ - - // As the SetUpFixture, the OneTimeTearDown of this class is executed after - // all tests have run. - [SetUpFixture] - public class GlobalTestsSetup - { - [OneTimeTearDown] - public void FinalCleanup() - { - if (PythonEngine.IsInitialized) - { - PythonEngine.Shutdown(); - } - } - } -} diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj index a741a589e..92d55a7e0 100644 --- a/src/embed_tests/Python.EmbeddingTest.15.csproj +++ b/src/embed_tests/Python.EmbeddingTest.15.csproj @@ -29,7 +29,6 @@ XPLAT $(DefineConstants);$(CustomDefineConstants);$(BaseDefineConstants); $(DefineConstants);NETCOREAPP - $(DefineConstants);NETSTANDARD $(DefineConstants);TRACE;DEBUG $(NuGetPackageRoot)\microsoft.targetingpack.netframework.v4.5\1.0.1\lib\net45\ diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index 6aa48becc..c2b323979 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -1,128 +1,127 @@ - - - - Debug - AnyCPU - {4165C59D-2822-499F-A6DB-EACA4C331EB5} - Library - Python.EmbeddingTest - Python.EmbeddingTest - bin\Python.EmbeddingTest.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - 6 - true - prompt - - - x86 - - - x64 - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - true - DEBUG;TRACE - full - - - - - true - pdbonly - - - - - ..\..\packages\NUnit.3.7.1\lib\net40\nunit.framework.dll - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Python.Runtime - - - - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - - + + + + Debug + AnyCPU + {4165C59D-2822-499F-A6DB-EACA4C331EB5} + Library + Python.EmbeddingTest + Python.EmbeddingTest + bin\Python.EmbeddingTest.xml + bin\ + v4.5.2 + + 1591 + ..\..\ + $(SolutionDir)\bin\ + 6 + true + prompt + + + x86 + + + x64 + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + true + DEBUG;TRACE + full + + + + + true + pdbonly + + + + + ..\..\packages\NUnit.3.7.1\lib\net40\nunit.framework.dll + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {097B4AC0-74E9-4C58-BCF8-C69746EC8271} + Python.Runtime + + + + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + + \ No newline at end of file diff --git a/src/embed_tests/TestConverter.cs b/src/embed_tests/TestConverter.cs index caaec311b..0f35570de 100644 --- a/src/embed_tests/TestConverter.cs +++ b/src/embed_tests/TestConverter.cs @@ -1,3 +1,5 @@ +using System; +using System.Collections.Generic; using NUnit.Framework; using Python.Runtime; @@ -17,6 +19,30 @@ public void Dispose() PythonEngine.Shutdown(); } + [Test] + public void ConvertListRoundTrip() + { + var list = new List { typeof(decimal), typeof(int) }; + var py = list.ToPython(); + object result; + var converted = Converter.ToManaged(py.Handle, typeof(List), out result, false); + + Assert.IsTrue(converted); + Assert.AreEqual(result, list); + } + + [Test] + public void ConvertPyListToArray() + { + var array = new List { typeof(decimal), typeof(int) }; + var py = array.ToPython(); + object result; + var converted = Converter.ToManaged(py.Handle, typeof(Type[]), out result, false); + + Assert.IsTrue(converted); + Assert.AreEqual(result, array); + } + [Test] public void TestConvertSingleToManaged( [Values(float.PositiveInfinity, float.NegativeInfinity, float.MinValue, float.MaxValue, float.NaN, diff --git a/src/embed_tests/TestDomainReload.cs b/src/embed_tests/TestDomainReload.cs deleted file mode 100644 index b162d4eb0..000000000 --- a/src/embed_tests/TestDomainReload.cs +++ /dev/null @@ -1,239 +0,0 @@ -using System; -using System.CodeDom.Compiler; -using System.Reflection; -using NUnit.Framework; -using Python.Runtime; - -// -// This test case is disabled on .NET Standard because it doesn't have all the -// APIs we use. We could work around that, but .NET Core doesn't implement -// domain creation, so it's not worth it. -// -// Unfortunately this means no continuous integration testing for this case. -// -#if !NETSTANDARD && !NETCOREAPP -namespace Python.EmbeddingTest -{ - class TestDomainReload - { - /// - /// Test that the python runtime can survive a C# domain reload without crashing. - /// - /// At the time this test was written, there was a very annoying - /// seemingly random crash bug when integrating pythonnet into Unity. - /// - /// The repro steps that David Lassonde, Viktoria Kovecses and - /// Benoit Hudson eventually worked out: - /// 1. Write a HelloWorld.cs script that uses Python.Runtime to access - /// some C# data from python: C# calls python, which calls C#. - /// 2. Execute the script (e.g. make it a MenuItem and click it). - /// 3. Touch HelloWorld.cs on disk, forcing Unity to recompile scripts. - /// 4. Wait several seconds for Unity to be done recompiling and - /// reloading the C# domain. - /// 5. Make python run the gc (e.g. by calling gc.collect()). - /// - /// The reason: - /// A. In step 2, Python.Runtime registers a bunch of new types with - /// their tp_traverse slot pointing to managed code, and allocates - /// some objects of those types. - /// B. In step 4, Unity unloads the C# domain. That frees the managed - /// code. But at the time of the crash investigation, pythonnet - /// leaked the python side of the objects allocated in step 1. - /// C. In step 5, python sees some pythonnet objects in its gc list of - /// potentially-leaked objects. It calls tp_traverse on those objects. - /// But tp_traverse was freed in step 3 => CRASH. - /// - /// This test distills what's going on without needing Unity around (we'd see - /// similar behaviour if we were using pythonnet on a .NET web server that did - /// a hot reload). - /// - [Test] - public static void DomainReloadAndGC() - { - // We're set up to run in the directory that includes the bin directory. - System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); - - Assembly pythonRunner1 = BuildAssembly("test1"); - RunAssemblyAndUnload(pythonRunner1, "test1"); - - // Verify that python is not initialized even though we ran it. - Assert.That(Runtime.Runtime.Py_IsInitialized(), Is.Zero); - - // This caused a crash because objects allocated in pythonRunner1 - // still existed in memory, but the code to do python GC on those - // objects is gone. - Assembly pythonRunner2 = BuildAssembly("test2"); - RunAssemblyAndUnload(pythonRunner2, "test2"); - } - - // - // The code we'll test. All that really matters is - // using GIL { Python.Exec(pyScript); } - // but the rest is useful for debugging. - // - // What matters in the python code is gc.collect and clr.AddReference. - // - // Note that the language version is 2.0, so no $"foo{bar}" syntax. - // - const string TestCode = @" - using Python.Runtime; - using System; - class PythonRunner { - public static void RunPython() { - AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; - string name = AppDomain.CurrentDomain.FriendlyName; - Console.WriteLine(string.Format(""[{0} in .NET] In PythonRunner.RunPython"", name)); - using (Py.GIL()) { - try { - var pyScript = string.Format(""import clr\n"" - + ""print('[{0} in python] imported clr')\n"" - + ""clr.AddReference('System')\n"" - + ""print('[{0} in python] allocated a clr object')\n"" - + ""import gc\n"" - + ""gc.collect()\n"" - + ""print('[{0} in python] collected garbage')\n"", - name); - PythonEngine.Exec(pyScript); - } catch(Exception e) { - Console.WriteLine(string.Format(""[{0} in .NET] Caught exception: {1}"", name, e)); - } - } - } - static void OnDomainUnload(object sender, EventArgs e) { - System.Console.WriteLine(string.Format(""[{0} in .NET] unloading"", AppDomain.CurrentDomain.FriendlyName)); - } - }"; - - - /// - /// Build an assembly out of the source code above. - /// - /// This creates a file .dll in order - /// to support the statement "proxy.theAssembly = assembly" below. - /// That statement needs a file, can't run via memory. - /// - static Assembly BuildAssembly(string assemblyName) - { - var provider = CodeDomProvider.CreateProvider("CSharp"); - - var compilerparams = new CompilerParameters(); - compilerparams.ReferencedAssemblies.Add("Python.Runtime.dll"); - compilerparams.GenerateExecutable = false; - compilerparams.GenerateInMemory = false; - compilerparams.IncludeDebugInformation = false; - compilerparams.OutputAssembly = assemblyName; - - var results = provider.CompileAssemblyFromSource(compilerparams, TestCode); - if (results.Errors.HasErrors) - { - var errors = new System.Text.StringBuilder("Compiler Errors:\n"); - foreach (CompilerError error in results.Errors) - { - errors.AppendFormat("Line {0},{1}\t: {2}\n", - error.Line, error.Column, error.ErrorText); - } - throw new Exception(errors.ToString()); - } - else - { - return results.CompiledAssembly; - } - } - - /// - /// This is a magic incantation required to run code in an application - /// domain other than the current one. - /// - class Proxy : MarshalByRefObject - { - Assembly theAssembly = null; - - public void InitAssembly(string assemblyPath) - { - theAssembly = Assembly.LoadFile(System.IO.Path.GetFullPath(assemblyPath)); - } - - public void RunPython() - { - Console.WriteLine("[Proxy] Entering RunPython"); - - // Call into the new assembly. Will execute Python code - var pythonrunner = theAssembly.GetType("PythonRunner"); - var runPythonMethod = pythonrunner.GetMethod("RunPython"); - runPythonMethod.Invoke(null, new object[] { }); - - Console.WriteLine("[Proxy] Leaving RunPython"); - } - } - - /// - /// Create a domain, run the assembly in it (the RunPython function), - /// and unload the domain. - /// - static void RunAssemblyAndUnload(Assembly assembly, string assemblyName) - { - Console.WriteLine($"[Program.Main] === creating domain for assembly {assembly.FullName}"); - - // Create the domain. Make sure to set PrivateBinPath to a relative - // path from the CWD (namely, 'bin'). - // See https://stackoverflow.com/questions/24760543/createinstanceandunwrap-in-another-domain - var currentDomain = AppDomain.CurrentDomain; - var domainsetup = new AppDomainSetup() - { - ApplicationBase = currentDomain.SetupInformation.ApplicationBase, - ConfigurationFile = currentDomain.SetupInformation.ConfigurationFile, - LoaderOptimization = LoaderOptimization.SingleDomain, - PrivateBinPath = "." - }; - var domain = AppDomain.CreateDomain( - $"My Domain {assemblyName}", - currentDomain.Evidence, - domainsetup); - - // Create a Proxy object in the new domain, where we want the - // assembly (and Python .NET) to reside - Type type = typeof(Proxy); - System.IO.Directory.SetCurrentDirectory(AppDomain.CurrentDomain.BaseDirectory); - var theProxy = (Proxy)domain.CreateInstanceAndUnwrap( - type.Assembly.FullName, - type.FullName); - - // From now on use the Proxy to call into the new assembly - theProxy.InitAssembly(assemblyName); - theProxy.RunPython(); - - Console.WriteLine($"[Program.Main] Before Domain Unload on {assembly.FullName}"); - AppDomain.Unload(domain); - Console.WriteLine($"[Program.Main] After Domain Unload on {assembly.FullName}"); - - // Validate that the assembly does not exist anymore - try - { - Console.WriteLine($"[Program.Main] The Proxy object is valid ({theProxy}). Unexpected domain unload behavior"); - } - catch (Exception) - { - Console.WriteLine("[Program.Main] The Proxy object is not valid anymore, domain unload complete."); - } - } - - /// - /// Resolves the assembly. Why doesn't this just work normally? - /// - static Assembly ResolveAssembly(object sender, ResolveEventArgs args) - { - var loadedAssemblies = AppDomain.CurrentDomain.GetAssemblies(); - - foreach (var assembly in loadedAssemblies) - { - if (assembly.FullName == args.Name) - { - return assembly; - } - } - - return null; - } - } -} -#endif diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs new file mode 100644 index 000000000..53838f315 --- /dev/null +++ b/src/embed_tests/TestFinalizer.cs @@ -0,0 +1,241 @@ +using NUnit.Framework; +using Python.Runtime; +using System; +using System.Linq; +using System.Threading; + +namespace Python.EmbeddingTest +{ + public class TestFinalizer + { + private int _oldThreshold; + + [SetUp] + public void SetUp() + { + _oldThreshold = Finalizer.Instance.Threshold; + PythonEngine.Initialize(); + Exceptions.Clear(); + } + + [TearDown] + public void TearDown() + { + Finalizer.Instance.Threshold = _oldThreshold; + PythonEngine.Shutdown(); + } + + private static void FullGCCollect() + { + GC.Collect(GC.MaxGeneration, GCCollectionMode.Forced); + GC.WaitForPendingFinalizers(); + } + + [Test] + public void CollectBasicObject() + { + Assert.IsTrue(Finalizer.Instance.Enable); + + Finalizer.Instance.Threshold = 1; + bool called = false; + var objectCount = 0; + EventHandler handler = (s, e) => + { + objectCount = e.ObjectCount; + called = true; + }; + + Assert.IsFalse(called); + Finalizer.Instance.CollectOnce += handler; + + WeakReference shortWeak; + WeakReference longWeak; + { + MakeAGarbage(out shortWeak, out longWeak); + } + FullGCCollect(); + // The object has been resurrected + Assert.IsFalse(shortWeak.IsAlive); + Assert.IsTrue(longWeak.IsAlive); + + { + var garbage = Finalizer.Instance.GetCollectedObjects(); + Assert.NotZero(garbage.Count); + Assert.IsTrue(garbage.Any(T => ReferenceEquals(T.Target, longWeak.Target))); + } + try + { + Finalizer.Instance.Collect(forceDispose: false); + } + finally + { + Finalizer.Instance.CollectOnce -= handler; + } + Assert.IsTrue(called); + Assert.GreaterOrEqual(objectCount, 1); + } + + private static void MakeAGarbage(out WeakReference shortWeak, out WeakReference longWeak) + { + PyLong obj = new PyLong(1024); + shortWeak = new WeakReference(obj); + longWeak = new WeakReference(obj, true); + obj = null; + } + + private static long CompareWithFinalizerOn(PyObject pyCollect, bool enbale) + { + // Must larger than 512 bytes make sure Python use + string str = new string('1', 1024); + Finalizer.Instance.Enable = true; + FullGCCollect(); + FullGCCollect(); + pyCollect.Invoke(); + Finalizer.Instance.Collect(); + Finalizer.Instance.Enable = enbale; + + // Estimate unmanaged memory size + long before = Environment.WorkingSet - GC.GetTotalMemory(true); + for (int i = 0; i < 10000; i++) + { + // Memory will leak when disable Finalizer + new PyString(str); + } + FullGCCollect(); + FullGCCollect(); + pyCollect.Invoke(); + if (enbale) + { + Finalizer.Instance.Collect(); + } + + FullGCCollect(); + FullGCCollect(); + long after = Environment.WorkingSet - GC.GetTotalMemory(true); + return after - before; + + } + + /// + /// Because of two vms both have their memory manager, + /// this test only prove the finalizer has take effect. + /// + [Test] + [Ignore("Too many uncertainties, only manual on when debugging")] + public void SimpleTestMemory() + { + bool oldState = Finalizer.Instance.Enable; + try + { + using (PyObject gcModule = PythonEngine.ImportModule("gc")) + using (PyObject pyCollect = gcModule.GetAttr("collect")) + { + long span1 = CompareWithFinalizerOn(pyCollect, false); + long span2 = CompareWithFinalizerOn(pyCollect, true); + Assert.Less(span2, span1); + } + } + finally + { + Finalizer.Instance.Enable = oldState; + } + } + + class MyPyObject : PyObject + { + public MyPyObject(IntPtr op) : base(op) + { + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + GC.SuppressFinalize(this); + throw new Exception("MyPyObject"); + } + internal static void CreateMyPyObject(IntPtr op) + { + Runtime.Runtime.XIncref(op); + new MyPyObject(op); + } + } + + [Test] + public void ErrorHandling() + { + bool called = false; + var errorMessage = ""; + EventHandler handleFunc = (sender, args) => + { + called = true; + errorMessage = args.Error.Message; + }; + Finalizer.Instance.Threshold = 1; + Finalizer.Instance.ErrorHandler += handleFunc; + try + { + WeakReference shortWeak; + WeakReference longWeak; + { + MakeAGarbage(out shortWeak, out longWeak); + var obj = (PyLong)longWeak.Target; + IntPtr handle = obj.Handle; + shortWeak = null; + longWeak = null; + MyPyObject.CreateMyPyObject(handle); + obj.Dispose(); + obj = null; + } + FullGCCollect(); + Finalizer.Instance.Collect(); + Assert.IsTrue(called); + } + finally + { + Finalizer.Instance.ErrorHandler -= handleFunc; + } + Assert.AreEqual(errorMessage, "MyPyObject"); + } + + [Test] + public void ValidateRefCount() + { + if (!Finalizer.Instance.RefCountValidationEnabled) + { + Assert.Pass("Only run with FINALIZER_CHECK"); + } + IntPtr ptr = IntPtr.Zero; + bool called = false; + Finalizer.IncorrectRefCntHandler handler = (s, e) => + { + called = true; + Assert.AreEqual(ptr, e.Handle); + Assert.AreEqual(2, e.ImpactedObjects.Count); + // Fix for this test, don't do this on general environment + Runtime.Runtime.XIncref(e.Handle); + return false; + }; + Finalizer.Instance.IncorrectRefCntResolver += handler; + try + { + ptr = CreateStringGarbage(); + FullGCCollect(); + Assert.Throws(() => Finalizer.Instance.Collect()); + Assert.IsTrue(called); + } + finally + { + Finalizer.Instance.IncorrectRefCntResolver -= handler; + } + } + + private static IntPtr CreateStringGarbage() + { + PyString s1 = new PyString("test_string"); + // s2 steal a reference from s1 + PyString s2 = new PyString(s1.Handle); + return s1.Handle; + } + + } +} diff --git a/src/embed_tests/TestMethodBinder.cs b/src/embed_tests/TestMethodBinder.cs new file mode 100644 index 000000000..ba4096776 --- /dev/null +++ b/src/embed_tests/TestMethodBinder.cs @@ -0,0 +1,151 @@ +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestMethodBinder + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + [Test] + public void ImplicitConversionToString() + { + // create instance of python model + dynamic pyMethodCall = PythonEngine.ModuleFromString("module", @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") + +from Python.EmbeddingTest import * + +class PythonModel(TestMethodBinder.CSharpModel): + def MethodCall(self): + return self.OnlyString(TestMethodBinder.TestImplicitConversion()) +").GetAttr("PythonModel").Invoke(); + + using (Py.GIL()) + { + var data = (string)pyMethodCall.MethodCall(); + // we assert implicit conversion took place + Assert.AreEqual("OnlyString impl: implicit to string", data); + } + } + + [Test] + public void ImplicitConversionToClass() + { + // create instance of python model + dynamic pyMethodCall = PythonEngine.ModuleFromString("module", @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") + +from Python.EmbeddingTest import * + +class PythonModel(TestMethodBinder.CSharpModel): + def MethodCall(self): + return self.OnlyClass('input string') +").GetAttr("PythonModel").Invoke(); + + using (Py.GIL()) + { + var data = (string)pyMethodCall.MethodCall(); + // we assert implicit conversion took place + Assert.AreEqual("OnlyClass impl", data); + } + } + + [Test] + public void WillAvoidUsingImplicitConversionIfPossible_String() + { + // create instance of python model + dynamic pyMethodCall = PythonEngine.ModuleFromString("module", @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") + +from Python.EmbeddingTest import * + +class PythonModel(TestMethodBinder.CSharpModel): + def MethodCall(self): + return self.InvokeModel('input string') +").GetAttr("PythonModel").Invoke(); + + using (Py.GIL()) + { + var data = (string)pyMethodCall.MethodCall(); + // we assert no implicit conversion took place + Assert.AreEqual("string impl: input string", data); + } + } + + [Test] + public void WillAvoidUsingImplicitConversionIfPossible_Class() + { + // create instance of python model + dynamic pyMethodCall = PythonEngine.ModuleFromString("module", @" +from clr import AddReference +AddReference(""System"") +AddReference(""Python.EmbeddingTest"") + +from Python.EmbeddingTest import * + +class PythonModel(TestMethodBinder.CSharpModel): + def MethodCall(self): + return self.InvokeModel(TestMethodBinder.TestImplicitConversion()) +").GetAttr("PythonModel").Invoke(); + + using (Py.GIL()) + { + var data = (string)pyMethodCall.MethodCall(); + // we assert no implicit conversion took place + Assert.AreEqual("TestImplicitConversion impl", data); + } + } + + public class CSharpModel + { + public virtual string OnlyClass(TestImplicitConversion data) + { + return "OnlyClass impl"; + } + + public virtual string OnlyString(string data) + { + return "OnlyString impl: " + data; + } + + public virtual string InvokeModel(string data) + { + return "string impl: " + data; + } + + public virtual string InvokeModel(TestImplicitConversion data) + { + return "TestImplicitConversion impl"; + } + } + + public class TestImplicitConversion + { + public static implicit operator string(TestImplicitConversion symbol) + { + return "implicit to string"; + } + public static implicit operator TestImplicitConversion(string symbol) + { + return new TestImplicitConversion(); + } + } + } +} diff --git a/src/embed_tests/TestNamedArguments.cs b/src/embed_tests/TestNamedArguments.cs index 31f2ea1d2..1d7076956 100644 --- a/src/embed_tests/TestNamedArguments.cs +++ b/src/embed_tests/TestNamedArguments.cs @@ -1,4 +1,4 @@ -using System; +using System; using NUnit.Framework; using Python.Runtime; diff --git a/src/embed_tests/TestPyAnsiString.cs b/src/embed_tests/TestPyAnsiString.cs index 9ba7d6cc6..b4a965ff7 100644 --- a/src/embed_tests/TestPyAnsiString.cs +++ b/src/embed_tests/TestPyAnsiString.cs @@ -63,6 +63,7 @@ public void TestCtorPtr() const string expected = "foo"; var t = new PyAnsiString(expected); + Runtime.Runtime.XIncref(t.Handle); var actual = new PyAnsiString(t.Handle); Assert.AreEqual(expected, actual.ToString()); diff --git a/src/embed_tests/TestPyFloat.cs b/src/embed_tests/TestPyFloat.cs index f2c85a77f..94e7026c7 100644 --- a/src/embed_tests/TestPyFloat.cs +++ b/src/embed_tests/TestPyFloat.cs @@ -25,6 +25,7 @@ public void Dispose() public void IntPtrCtor() { var i = new PyFloat(1); + Runtime.Runtime.XIncref(i.Handle); var ii = new PyFloat(i.Handle); Assert.AreEqual(i.Handle, ii.Handle); } diff --git a/src/embed_tests/TestPyInt.cs b/src/embed_tests/TestPyInt.cs index 4117336d8..005ab466d 100644 --- a/src/embed_tests/TestPyInt.cs +++ b/src/embed_tests/TestPyInt.cs @@ -86,6 +86,7 @@ public void TestCtorSByte() public void TestCtorPtr() { var i = new PyInt(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyInt(i.Handle); Assert.AreEqual(5, a.ToInt32()); } @@ -94,6 +95,7 @@ public void TestCtorPtr() public void TestCtorPyObject() { var i = new PyInt(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyInt(i); Assert.AreEqual(5, a.ToInt32()); } diff --git a/src/embed_tests/TestPyLong.cs b/src/embed_tests/TestPyLong.cs index fe3e13ef5..3c155f315 100644 --- a/src/embed_tests/TestPyLong.cs +++ b/src/embed_tests/TestPyLong.cs @@ -102,6 +102,7 @@ public void TestCtorDouble() public void TestCtorPtr() { var i = new PyLong(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyLong(i.Handle); Assert.AreEqual(5, a.ToInt32()); } @@ -110,6 +111,7 @@ public void TestCtorPtr() public void TestCtorPyObject() { var i = new PyLong(5); + Runtime.Runtime.XIncref(i.Handle); var a = new PyLong(i); Assert.AreEqual(5, a.ToInt32()); } diff --git a/src/embed_tests/TestPyObject.cs b/src/embed_tests/TestPyObject.cs index 65ac20e9a..d794ce06e 100644 --- a/src/embed_tests/TestPyObject.cs +++ b/src/embed_tests/TestPyObject.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using NUnit.Framework; diff --git a/src/embed_tests/TestPyScope.cs b/src/embed_tests/TestPyScope.cs index 49c15a3a1..21c0d2b3f 100644 --- a/src/embed_tests/TestPyScope.cs +++ b/src/embed_tests/TestPyScope.cs @@ -293,24 +293,27 @@ public void TestImportScopeByName() [Test] public void TestVariables() { - (ps.Variables() as dynamic)["ee"] = new PyInt(200); - var a0 = ps.Get("ee"); - Assert.AreEqual(200, a0); + using (Py.GIL()) + { + (ps.Variables() as dynamic)["ee"] = new PyInt(200); + var a0 = ps.Get("ee"); + Assert.AreEqual(200, a0); - ps.Exec("locals()['ee'] = 210"); - var a1 = ps.Get("ee"); - Assert.AreEqual(210, a1); + ps.Exec("locals()['ee'] = 210"); + var a1 = ps.Get("ee"); + Assert.AreEqual(210, a1); - ps.Exec("globals()['ee'] = 220"); - var a2 = ps.Get("ee"); - Assert.AreEqual(220, a2); + ps.Exec("globals()['ee'] = 220"); + var a2 = ps.Get("ee"); + Assert.AreEqual(220, a2); - using (var item = ps.Variables()) - { - item["ee"] = new PyInt(230); + using (var item = ps.Variables()) + { + item["ee"] = new PyInt(230); + } + var a3 = ps.Get("ee"); + Assert.AreEqual(230, a3); } - var a3 = ps.Get("ee"); - Assert.AreEqual(230, a3); } /// @@ -324,49 +327,55 @@ public void TestThread() //should be removed. dynamic _ps = ps; var ts = PythonEngine.BeginAllowThreads(); - using (Py.GIL()) - { - _ps.res = 0; - _ps.bb = 100; - _ps.th_cnt = 0; - //add function to the scope - //can be call many times, more efficient than ast - ps.Exec( - "def update():\n" + - " global res, th_cnt\n" + - " res += bb + 1\n" + - " th_cnt += 1\n" - ); - } - int th_cnt = 3; - for (int i =0; i< th_cnt; i++) + try { - System.Threading.Thread th = new System.Threading.Thread(()=> + using (Py.GIL()) + { + _ps.res = 0; + _ps.bb = 100; + _ps.th_cnt = 0; + //add function to the scope + //can be call many times, more efficient than ast + ps.Exec( + "def update():\n" + + " global res, th_cnt\n" + + " res += bb + 1\n" + + " th_cnt += 1\n" + ); + } + int th_cnt = 3; + for (int i = 0; i < th_cnt; i++) + { + System.Threading.Thread th = new System.Threading.Thread(() => + { + using (Py.GIL()) + { + //ps.GetVariable("update")(); //call the scope function dynamicly + _ps.update(); + } + }); + th.Start(); + } + //equivalent to Thread.Join, make the main thread join the GIL competition + int cnt = 0; + while (cnt != th_cnt) { using (Py.GIL()) { - //ps.GetVariable("update")(); //call the scope function dynamicly - _ps.update(); + cnt = ps.Get("th_cnt"); } - }); - th.Start(); - } - //equivalent to Thread.Join, make the main thread join the GIL competition - int cnt = 0; - while(cnt != th_cnt) - { + System.Threading.Thread.Sleep(10); + } using (Py.GIL()) { - cnt = ps.Get("th_cnt"); + var result = ps.Get("res"); + Assert.AreEqual(101 * th_cnt, result); } - System.Threading.Thread.Sleep(10); } - using (Py.GIL()) + finally { - var result = ps.Get("res"); - Assert.AreEqual(101* th_cnt, result); + PythonEngine.EndAllowThreads(ts); } - PythonEngine.EndAllowThreads(ts); } } } diff --git a/src/embed_tests/TestPySequence.cs b/src/embed_tests/TestPySequence.cs index 7c175b1ce..1e3ebf144 100644 --- a/src/embed_tests/TestPySequence.cs +++ b/src/embed_tests/TestPySequence.cs @@ -69,8 +69,10 @@ public void TestRepeat() PyObject actual = t1.Repeat(3); Assert.AreEqual("FooFooFoo", actual.ToString()); - actual = t1.Repeat(-3); - Assert.AreEqual("", actual.ToString()); + // On 32 bit system this argument should be int, but on the 64 bit system this should be long value. + // This works on the Framework 4.0 accidentally, it should produce out of memory! + // actual = t1.Repeat(-3); + // Assert.AreEqual("", actual.ToString()); } [Test] diff --git a/src/embed_tests/TestPyString.cs b/src/embed_tests/TestPyString.cs index 9d1cdb0e9..0de436e35 100644 --- a/src/embed_tests/TestPyString.cs +++ b/src/embed_tests/TestPyString.cs @@ -64,6 +64,7 @@ public void TestCtorPtr() const string expected = "foo"; var t = new PyString(expected); + Runtime.Runtime.XIncref(t.Handle); var actual = new PyString(t.Handle); Assert.AreEqual(expected, actual.ToString()); diff --git a/src/embed_tests/TestPyWith.cs b/src/embed_tests/TestPyWith.cs index 0c4e9023f..fd3f8e662 100644 --- a/src/embed_tests/TestPyWith.cs +++ b/src/embed_tests/TestPyWith.cs @@ -1,4 +1,4 @@ -using System; +using System; using NUnit.Framework; using Python.Runtime; diff --git a/src/embed_tests/TestPythonEngineProperties.cs b/src/embed_tests/TestPythonEngineProperties.cs index 243349b82..01c6ae7e3 100644 --- a/src/embed_tests/TestPythonEngineProperties.cs +++ b/src/embed_tests/TestPythonEngineProperties.cs @@ -109,41 +109,18 @@ public static void GetPythonHomeDefault() [Test] public void SetPythonHome() { - // We needs to ensure that engine was started and shutdown at least once before setting dummy home. - // Otherwise engine will not run with dummy path with random problem. - if (!PythonEngine.IsInitialized) - { - PythonEngine.Initialize(); - } - - PythonEngine.Shutdown(); - - var pythonHomeBackup = PythonEngine.PythonHome; - var pythonHome = "/dummypath/"; PythonEngine.PythonHome = pythonHome; PythonEngine.Initialize(); + Assert.AreEqual(pythonHome, PythonEngine.PythonHome); PythonEngine.Shutdown(); - - // Restoring valid pythonhome. - PythonEngine.PythonHome = pythonHomeBackup; } [Test] public void SetPythonHomeTwice() { - // We needs to ensure that engine was started and shutdown at least once before setting dummy home. - // Otherwise engine will not run with dummy path with random problem. - if (!PythonEngine.IsInitialized) - { - PythonEngine.Initialize(); - } - PythonEngine.Shutdown(); - - var pythonHomeBackup = PythonEngine.PythonHome; - var pythonHome = "/dummypath/"; PythonEngine.PythonHome = "/dummypath2/"; @@ -152,20 +129,11 @@ public void SetPythonHomeTwice() Assert.AreEqual(pythonHome, PythonEngine.PythonHome); PythonEngine.Shutdown(); - - PythonEngine.PythonHome = pythonHomeBackup; } [Test] public void SetProgramName() { - if (PythonEngine.IsInitialized) - { - PythonEngine.Shutdown(); - } - - var programNameBackup = PythonEngine.ProgramName; - var programName = "FooBar"; PythonEngine.ProgramName = programName; @@ -173,8 +141,6 @@ public void SetProgramName() Assert.AreEqual(programName, PythonEngine.ProgramName); PythonEngine.Shutdown(); - - PythonEngine.ProgramName = programNameBackup; } [Test] @@ -190,7 +156,7 @@ public void SetPythonPath() string path = PythonEngine.PythonPath; PythonEngine.Shutdown(); - PythonEngine.PythonPath = path; + PythonEngine.ProgramName = path; PythonEngine.Initialize(); Assert.AreEqual(path, PythonEngine.PythonPath); @@ -205,6 +171,7 @@ public void SetPythonPathExceptionOn27() Assert.Pass(); } + // Get previous path to avoid crashing Python PythonEngine.Initialize(); string path = PythonEngine.PythonPath; PythonEngine.Shutdown(); diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index ac1fa1ac0..2e0598da7 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -1,4 +1,4 @@ -using System; +using System; using NUnit.Framework; using Python.Runtime; @@ -6,40 +6,10 @@ namespace Python.EmbeddingTest { public class TestRuntime { - [OneTimeSetUp] - public void SetUp() - { - // We needs to ensure that no any engines are running. - if (PythonEngine.IsInitialized) - { - PythonEngine.Shutdown(); - } - } - - /// - /// Test the cache of the information from the platform module. - /// - /// Test fails on platforms we haven't implemented yet. - /// - [Test] - public static void PlatformCache() - { - Runtime.Runtime.Initialize(); - - Assert.That(Runtime.Runtime.Machine, Is.Not.EqualTo(Runtime.Runtime.MachineType.Other)); - Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.MachineName)); - - Assert.That(Runtime.Runtime.OperatingSystem, Is.Not.EqualTo(Runtime.Runtime.OperatingSystemType.Other)); - Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.OperatingSystemName)); - - // Don't shut down the runtime: if the python engine was initialized - // but not shut down by another test, we'd end up in a bad state. - } - [Test] public static void Py_IsInitializedValue() { - Runtime.Runtime.Py_Finalize(); + Runtime.Runtime.Py_Finalize(); // In case another test left it on. Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); Runtime.Runtime.Py_Initialize(); Assert.AreEqual(1, Runtime.Runtime.Py_IsInitialized()); diff --git a/src/embed_tests/TestTypeManager.cs b/src/embed_tests/TestTypeManager.cs deleted file mode 100644 index a4ef86913..000000000 --- a/src/embed_tests/TestTypeManager.cs +++ /dev/null @@ -1,65 +0,0 @@ -using NUnit.Framework; -using Python.Runtime; -using System.Runtime.InteropServices; - -namespace Python.EmbeddingTest -{ - class TestTypeManager - { - [SetUp] - public static void Init() - { - Runtime.Runtime.Initialize(); - } - - [TearDown] - public static void Fini() - { - // Don't shut down the runtime: if the python engine was initialized - // but not shut down by another test, we'd end up in a bad state. - } - - [Test] - public static void TestNativeCode() - { - Assert.That(() => { var _ = TypeManager.NativeCode.Active; }, Throws.Nothing); - Assert.That(TypeManager.NativeCode.Active.Code.Length, Is.GreaterThan(0)); - } - - [Test] - public static void TestMemoryMapping() - { - Assert.That(() => { var _ = TypeManager.CreateMemoryMapper(); }, Throws.Nothing); - var mapper = TypeManager.CreateMemoryMapper(); - - // Allocate a read-write page. - int len = 12; - var page = mapper.MapWriteable(len); - Assert.That(() => { Marshal.WriteInt64(page, 17); }, Throws.Nothing); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - - // Mark it read-execute. We can still read, haven't changed any values. - mapper.SetReadExec(page, len); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - - // Test that we can't write to the protected page. - // - // We can't actually test access protection under Microsoft - // versions of .NET, because AccessViolationException is assumed to - // mean we're in a corrupted state: - // https://stackoverflow.com/questions/3469368/how-to-handle-accessviolationexception - // - // We can test under Mono but it throws NRE instead of AccessViolationException. - // - // We can't use compiler flags because we compile with MONO_LINUX - // while running on the Microsoft .NET Core during continuous - // integration tests. - if (System.Type.GetType ("Mono.Runtime") != null) - { - // Mono throws NRE instead of AccessViolationException for some reason. - Assert.That(() => { Marshal.WriteInt64(page, 73); }, Throws.TypeOf()); - Assert.That(Marshal.ReadInt64(page), Is.EqualTo(17)); - } - } - } -} diff --git a/src/embed_tests/dynamic.cs b/src/embed_tests/dynamic.cs index 81345cee7..d75dc01d6 100644 --- a/src/embed_tests/dynamic.cs +++ b/src/embed_tests/dynamic.cs @@ -12,23 +12,13 @@ public class DynamicTest [SetUp] public void SetUp() { - try { - _gs = Py.GIL(); - } catch (Exception e) { - Console.WriteLine($"exception in SetUp: {e}"); - throw; - } + _gs = Py.GIL(); } [TearDown] public void Dispose() { - try { - _gs.Dispose(); - } catch(Exception e) { - Console.WriteLine($"exception in TearDown: {e}"); - throw; - } + _gs.Dispose(); } /// @@ -128,11 +118,10 @@ public void PassPyObjectInNet() sys.testattr2 = sys.testattr1; // Compare in Python - PythonEngine.RunSimpleString( + PyObject res = PythonEngine.RunString( "import sys\n" + "sys.testattr3 = sys.testattr1 is sys.testattr2\n" ); - Assert.AreEqual(sys.testattr3.ToString(), "True"); // Compare in .NET diff --git a/src/embed_tests/packages.config b/src/embed_tests/packages.config index 8c175f441..b5504a5e4 100644 --- a/src/embed_tests/packages.config +++ b/src/embed_tests/packages.config @@ -1,5 +1,5 @@ - + - - - + + + \ No newline at end of file diff --git a/src/embed_tests/pyinitialize.cs b/src/embed_tests/pyinitialize.cs index ea1d8d023..2f9aae2c7 100644 --- a/src/embed_tests/pyinitialize.cs +++ b/src/embed_tests/pyinitialize.cs @@ -74,65 +74,5 @@ public void ReInitialize() } PythonEngine.Shutdown(); } - - [Test] - public void TestScopeIsShutdown() - { - PythonEngine.Initialize(); - var scope = PyScopeManager.Global.Create("test"); - PythonEngine.Shutdown(); - Assert.That(PyScopeManager.Global.Contains("test"), Is.False); - } - - /// - /// Helper for testing the shutdown handlers. - /// - int shutdown_count = 0; - void OnShutdownIncrement() - { - shutdown_count++; - } - void OnShutdownDouble() - { - shutdown_count *= 2; - } - - /// - /// Test the shutdown handlers. - /// - [Test] - public void ShutdownHandlers() - { - // Test we can run one shutdown handler. - shutdown_count = 0; - PythonEngine.Initialize(); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.Shutdown(); - Assert.That(shutdown_count, Is.EqualTo(1)); - - // Test we can run multiple shutdown handlers in the right order. - shutdown_count = 4; - PythonEngine.Initialize(); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.AddShutdownHandler(OnShutdownDouble); - PythonEngine.Shutdown(); - // Correct: 4 * 2 + 1 = 9 - // Wrong: (4 + 1) * 2 = 10 - Assert.That(shutdown_count, Is.EqualTo(9)); - - // Test we can remove shutdown handlers, handling duplicates. - shutdown_count = 4; - PythonEngine.Initialize(); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.AddShutdownHandler(OnShutdownDouble); - PythonEngine.AddShutdownHandler(OnShutdownIncrement); - PythonEngine.AddShutdownHandler(OnShutdownDouble); - PythonEngine.RemoveShutdownHandler(OnShutdownDouble); - PythonEngine.Shutdown(); - // Correct: (4 + 1) * 2 + 1 + 1 = 12 - // Wrong: (4 * 2) + 1 + 1 + 1 = 11 - Assert.That(shutdown_count, Is.EqualTo(12)); - } } } diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj index 794645994..2497844bb 100644 --- a/src/runtime/Python.Runtime.15.csproj +++ b/src/runtime/Python.Runtime.15.csproj @@ -35,7 +35,7 @@ $(PYTHONNET_PY2_VERSION) PYTHON27 $(PYTHONNET_PY3_VERSION) - PYTHON37 + PYTHON36 $(PYTHONNET_WIN_DEFINE_CONSTANTS) UCS2 $(PYTHONNET_MONO_DEFINE_CONSTANTS) @@ -68,10 +68,10 @@ $(DefineConstants);PYTHON3;$(Python3Version);$(PythonMonoDefineConstants) - $(DefineConstants);PYTHON2;$(Python2Version);$(PythonMonoDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON2;$(Python2Version);$(PythonMonoDefineConstants);FINALIZER_CHECK;TRACE;DEBUG - $(DefineConstants);PYTHON3;$(Python3Version);$(PythonMonoDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON3;$(Python3Version);$(PythonMonoDefineConstants);FINALIZER_CHECK;TRACE;DEBUG $(DefineConstants);PYTHON2;$(Python2Version);$(PythonWinDefineConstants) @@ -80,10 +80,10 @@ $(DefineConstants);PYTHON3;$(Python3Version);$(PythonWinDefineConstants) - $(DefineConstants);PYTHON2;$(Python2Version);$(PythonWinDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON2;$(Python2Version);$(PythonWinDefineConstants);FINALIZER_CHECK;TRACE;DEBUG - $(DefineConstants);PYTHON3;$(Python3Version);$(PythonWinDefineConstants);TRACE;DEBUG + $(DefineConstants);PYTHON3;$(Python3Version);$(PythonWinDefineConstants);FINALIZER_CHECK;TRACE;DEBUG @@ -131,7 +131,7 @@ - + diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index fc155ca91..3ef2ad471 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,170 +1,171 @@ - - - - Debug - AnyCPU - {097B4AC0-74E9-4C58-BCF8-C69746EC8271} - Library - Python.Runtime - Python.Runtime - bin\Python.Runtime.xml - bin\ - v4.0 - - 1591 - ..\..\ - $(SolutionDir)\bin\ - Properties - 6 - true - false - ..\pythonnet.snk - - - - - - PYTHON2;PYTHON27;UCS4 - true - pdbonly - - - PYTHON3;PYTHON37;UCS4 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS4;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON37;UCS4;TRACE;DEBUG - false - full - - - PYTHON2;PYTHON27;UCS2 - true - pdbonly - - - PYTHON3;PYTHON37;UCS2 - true - pdbonly - - - true - PYTHON2;PYTHON27;UCS2;TRACE;DEBUG - false - full - - - true - PYTHON3;PYTHON37;UCS2;TRACE;DEBUG - false - full - - - - - - - - Properties\SharedAssemblyInfo.cs - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - clr.py - - - - - $(TargetPath) - $(TargetDir)$(TargetName).pdb - - - - - - + + + + Debug + AnyCPU + {097B4AC0-74E9-4C58-BCF8-C69746EC8271} + Library + Python.Runtime + Python.Runtime + bin\Python.Runtime.xml + bin\ + v4.5.2 + + 1591 + ..\..\ + $(SolutionDir)\bin\ + Properties + 6 + true + false + ..\pythonnet.snk + + + + + + PYTHON2;PYTHON27;UCS4 + true + pdbonly + + + PYTHON3;PYTHON36;UCS4 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS4;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON36;UCS4;TRACE;DEBUG + false + full + + + PYTHON2;PYTHON27;UCS2 + true + pdbonly + + + PYTHON3;PYTHON36;UCS2 + true + pdbonly + + + true + PYTHON2;PYTHON27;UCS2;TRACE;DEBUG + false + full + + + true + PYTHON3;PYTHON36;UCS2;TRACE;DEBUG + false + full + + + + + + + + + + Properties\SharedAssemblyInfo.cs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + clr.py + + + + + $(TargetPath) + $(TargetDir)$(TargetName).pdb + + + + + + \ No newline at end of file diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs index dc5f78608..dd4418cc9 100644 --- a/src/runtime/Util.cs +++ b/src/runtime/Util.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Runtime.InteropServices; namespace Python.Runtime diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index c37295704..a10688749 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -93,7 +93,7 @@ public static IntPtr mp_subscript(IntPtr ob, IntPtr idx) return IntPtr.Zero; } - var count = Runtime.PyTuple_Size(idx); + int count = Runtime.PyTuple_Size(idx); var args = new int[count]; @@ -186,7 +186,7 @@ public static int mp_ass_subscript(IntPtr ob, IntPtr idx, IntPtr v) return -1; } - var count = Runtime.PyTuple_Size(idx); + int count = Runtime.PyTuple_Size(idx); var args = new int[count]; for (var i = 0; i < count; i++) diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 3085bb639..a22db843d 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -1,5 +1,4 @@ using System; -using System.Collections; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; @@ -7,6 +6,7 @@ using System.Linq; using System.Reflection; using System.Threading; +using System.Threading.Tasks; namespace Python.Runtime { @@ -18,25 +18,16 @@ internal class AssemblyManager { // modified from event handlers below, potentially triggered from different .NET threads // therefore this should be a ConcurrentDictionary - // - // WARNING: Dangerous if cross-app domain usage is ever supported - // Reusing the dictionary with assemblies accross multiple initializations is problematic. - // Loading happens from CurrentDomain (see line 53). And if the first call is from AppDomain that is later unloaded, - // than it can end up referring to assemblies that are already unloaded (default behavior after unload appDomain - - // unless LoaderOptimization.MultiDomain is used); - // So for multidomain support it is better to have the dict. recreated for each app-domain initialization - private static ConcurrentDictionary> namespaces = - new ConcurrentDictionary>(); - //private static Dictionary> generics; - private static AssemblyLoadEventHandler lhandler; - private static ResolveEventHandler rhandler; + private static ConcurrentDictionary> namespaces; + private static ConcurrentDictionary assembliesNamesCache; + private static ConcurrentDictionary lookupTypeCache; + private static ConcurrentQueue assemblies; + private static int pendingAssemblies; // updated only under GIL? - private static Dictionary probed = new Dictionary(32); - - // modified from event handlers below, potentially triggered from different .NET threads - private static ConcurrentQueue assemblies; - internal static List pypath; + private static Dictionary probed; + private static List pypath; + private static Dictionary> filesInPath; private AssemblyManager() { @@ -49,30 +40,42 @@ private AssemblyManager() /// internal static void Initialize() { + namespaces = new ConcurrentDictionary>(); + assembliesNamesCache = new ConcurrentDictionary(); + lookupTypeCache = new ConcurrentDictionary(); + probed = new Dictionary(32); assemblies = new ConcurrentQueue(); pypath = new List(16); + filesInPath = new Dictionary>(); AppDomain domain = AppDomain.CurrentDomain; - lhandler = new AssemblyLoadEventHandler(AssemblyLoadHandler); - domain.AssemblyLoad += lhandler; - - rhandler = new ResolveEventHandler(ResolveHandler); - domain.AssemblyResolve += rhandler; + domain.AssemblyLoad += AssemblyLoadHandler; + domain.AssemblyResolve += ResolveHandler; - Assembly[] items = domain.GetAssemblies(); - foreach (Assembly a in items) + foreach (var assembly in domain.GetAssemblies()) { try { - ScanAssembly(a); - assemblies.Enqueue(a); + LaunchAssemblyLoader(assembly); } catch (Exception ex) { - Debug.WriteLine("Error scanning assembly {0}. {1}", a, ex); + Debug.WriteLine("Error scanning assembly {0}. {1}", assembly, ex); } } + + var safeCount = 0; + // lets wait until all assemblies are loaded + do + { + if (safeCount++ > 200) + { + throw new TimeoutException("Timeout while waiting for assemblies to load"); + } + + Thread.Sleep(50); + } while (pendingAssemblies > 0); } @@ -82,8 +85,8 @@ internal static void Initialize() internal static void Shutdown() { AppDomain domain = AppDomain.CurrentDomain; - domain.AssemblyLoad -= lhandler; - domain.AssemblyResolve -= rhandler; + domain.AssemblyLoad -= AssemblyLoadHandler; + domain.AssemblyResolve -= ResolveHandler; } @@ -94,13 +97,41 @@ internal static void Shutdown() /// so that we can know about assemblies that get loaded after the /// Python runtime is initialized. /// + /// Scanning assemblies here caused internal hangs when calling + /// private static void AssemblyLoadHandler(object ob, AssemblyLoadEventArgs args) { Assembly assembly = args.LoadedAssembly; - assemblies.Enqueue(assembly); - ScanAssembly(assembly); + LaunchAssemblyLoader(assembly); } + /// + /// Launches a new task that will load the provided assembly + /// + private static void LaunchAssemblyLoader(Assembly assembly) + { + if (assembly != null) + { + Interlocked.Increment(ref pendingAssemblies); + Task.Factory.StartNew(() => + { + try + { + if (assembliesNamesCache.TryAdd(assembly.GetName().Name, assembly)) + { + assemblies.Enqueue(assembly); + ScanAssembly(assembly); + } + } + catch + { + // pass + } + + Interlocked.Decrement(ref pendingAssemblies); + }); + } + } /// /// Event handler for assembly resolve events. This is needed because @@ -112,12 +143,12 @@ private static void AssemblyLoadHandler(object ob, AssemblyLoadEventArgs args) private static Assembly ResolveHandler(object ob, ResolveEventArgs args) { string name = args.Name.ToLower(); - foreach (Assembly a in assemblies) + foreach (var assembly in assemblies) { - string full = a.FullName.ToLower(); + var full = assembly.FullName.ToLower(); if (full.StartsWith(name)) { - return a; + return assembly; } } return LoadAssemblyPath(args.Name); @@ -138,24 +169,65 @@ private static Assembly ResolveHandler(object ob, ResolveEventArgs args) internal static void UpdatePath() { IntPtr list = Runtime.PySys_GetObject("path"); - var count = Runtime.PyList_Size(list); + int count = Runtime.PyList_Size(list); + var sep = Path.DirectorySeparatorChar; + if (count != pypath.Count) { pypath.Clear(); probed.Clear(); + // add first the current path + pypath.Add(""); for (var i = 0; i < count; i++) { IntPtr item = Runtime.PyList_GetItem(list, i); string path = Runtime.GetManagedString(item); if (path != null) { - pypath.Add(path); + pypath.Add(path == string.Empty ? path : path + sep); } } + + // for performance we will search for all files in each directory in the path once + Parallel.ForEach(pypath.Where(s => + { + try + { + lock (filesInPath) + { + // only search in directory if it exists and we haven't already analyzed it + return Directory.Exists(s) && !filesInPath.ContainsKey(s); + } + } + catch + { + // just in case, file operations can throw + } + return false; + }), path => + { + var container = new HashSet(); + try + { + foreach (var file in Directory.EnumerateFiles(path) + .Where(file => file.EndsWith(".dll") || file.EndsWith(".exe"))) + { + container.Add(Path.GetFileName(file)); + } + } + catch + { + // just in case, file operations can throw + } + + lock (filesInPath) + { + filesInPath[path] = container; + } + }); } } - /// /// Given an assembly name, try to find this assembly file using the /// PYTHONPATH. If not found, return null to indicate implicit load @@ -163,30 +235,17 @@ internal static void UpdatePath() /// public static string FindAssembly(string name) { - char sep = Path.DirectorySeparatorChar; - - foreach (string head in pypath) + foreach (var kvp in filesInPath) { - string path; - if (head == null || head.Length == 0) + var dll = $"{name}.dll"; + if (kvp.Value.Contains(dll)) { - path = name; + return kvp.Key + dll; } - else + var executable = $"{name}.exe"; + if (kvp.Value.Contains(executable)) { - path = head + sep + name; - } - - string temp = path + ".dll"; - if (File.Exists(temp)) - { - return temp; - } - - temp = path + ".exe"; - if (File.Exists(temp)) - { - return temp; + return kvp.Key + executable; } } return null; @@ -206,10 +265,7 @@ public static Assembly LoadAssembly(string name) } catch (Exception) { - //if (!(e is System.IO.FileNotFoundException)) - //{ - // throw; - //} + // ignored } return assembly; } @@ -220,7 +276,7 @@ public static Assembly LoadAssembly(string name) /// public static Assembly LoadAssemblyPath(string name) { - string path = FindAssembly(name); + var path = FindAssembly(name); Assembly assembly = null; if (path != null) { @@ -230,6 +286,7 @@ public static Assembly LoadAssemblyPath(string name) } catch (Exception) { + // ignored } } return assembly; @@ -257,6 +314,7 @@ public static Assembly LoadAssemblyFullPath(string name) } catch (Exception) { + // ignored } } } @@ -268,14 +326,8 @@ public static Assembly LoadAssemblyFullPath(string name) /// public static Assembly FindLoadedAssembly(string name) { - foreach (Assembly a in assemblies) - { - if (a.GetName().Name == name) - { - return a; - } - } - return null; + Assembly result; + return assembliesNamesCache.TryGetValue(name, out result) ? result : null; } /// @@ -312,10 +364,6 @@ public static bool LoadImplicit(string name, bool warn = true) { a = LoadAssemblyPath(s); } - if (a == null) - { - a = LoadAssembly(s); - } if (a != null && !assembliesSet.Contains(a)) { loaded = true; @@ -349,7 +397,9 @@ internal static void ScanAssembly(Assembly assembly) // A couple of things we want to do here: first, we want to // gather a list of all of the namespaces contributed to by // the assembly. - foreach (Type t in GetTypes(assembly)) + + Type[] types = assembly.GetTypes(); + foreach (Type t in types) { string ns = t.Namespace ?? ""; if (!namespaces.ContainsKey(ns)) @@ -359,13 +409,13 @@ internal static void ScanAssembly(Assembly assembly) for (var n = 0; n < names.Length; n++) { s = n == 0 ? names[0] : s + "." + names[n]; - namespaces.TryAdd(s, new ConcurrentDictionary()); + namespaces.TryAdd(s, new ConcurrentDictionary()); } } if (ns != null) { - namespaces[ns].TryAdd(assembly, string.Empty); + namespaces[ns].TryAdd(assembly, 1); } if (ns != null && t.IsGenericTypeDefinition) @@ -423,9 +473,10 @@ public static List GetNames(string nsname) { foreach (Assembly a in namespaces[nsname].Keys) { - foreach (Type t in GetTypes(a)) + Type[] types = a.GetTypes(); + foreach (Type t in types) { - if ((t.Namespace ?? "") == nsname && !t.IsNested) + if ((t.Namespace ?? "") == nsname) { names.Add(t.Name); } @@ -454,42 +505,21 @@ public static List GetNames(string nsname) /// public static Type LookupType(string qname) { + Type type; + if (lookupTypeCache.TryGetValue(qname, out type)) + { + return type; + } foreach (Assembly assembly in assemblies) { - Type type = assembly.GetType(qname); + type = assembly.GetType(qname); if (type != null) { + lookupTypeCache[qname] = type; return type; } } return null; } - - internal static Type[] GetTypes(Assembly a) - { - if (a.IsDynamic) - { - try - { - return a.GetTypes(); - } - catch (ReflectionTypeLoadException exc) - { - // Return all types that were successfully loaded - return exc.Types.Where(x => x != null).ToArray(); - } - } - else - { - try - { - return a.GetExportedTypes(); - } - catch (FileNotFoundException) - { - return new Type[0]; - } - } - } } -} \ No newline at end of file +} diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 5846fa82a..128e9795e 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -247,13 +247,31 @@ public static IntPtr tp_str(IntPtr ob) } + /// + /// Default implementations for required Python GC support. + /// + public static int tp_traverse(IntPtr ob, IntPtr func, IntPtr args) + { + return 0; + } + + public static int tp_clear(IntPtr ob) + { + return 0; + } + + public static int tp_is_gc(IntPtr type) + { + return 1; + } + /// /// Standard dealloc implementation for instances of reflected types. /// public static void tp_dealloc(IntPtr ob) { ManagedType self = GetManagedObject(ob); - IntPtr dict = Marshal.ReadIntPtr(ob, ObjectOffset.DictOffset(ob)); + IntPtr dict = Marshal.ReadIntPtr(ob, ObjectOffset.TypeDictOffset(self.tpHandle)); if (dict != IntPtr.Zero) { Runtime.XDecref(dict); diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index ec3734ea5..8ae96695f 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -3,7 +3,6 @@ using System.Linq; using System.Reflection; using System.Reflection.Emit; -using System.Resources; using System.Runtime.InteropServices; using System.Threading.Tasks; @@ -33,12 +32,6 @@ static ClassDerivedObject() moduleBuilders = new Dictionary, ModuleBuilder>(); } - public static void Reset() - { - assemblyBuilders = new Dictionary(); - moduleBuilders = new Dictionary, ModuleBuilder>(); - } - internal ClassDerivedObject(Type tp) : base(tp) { } @@ -877,7 +870,7 @@ public static void Finalize(IPythonDerivedType obj) // the C# object is being destroyed which must mean there are no more // references to the Python object as well so now we can dealloc the // python object. - IntPtr dict = Marshal.ReadIntPtr(self.pyHandle, ObjectOffset.DictOffset(self.pyHandle)); + IntPtr dict = Marshal.ReadIntPtr(self.pyHandle, ObjectOffset.TypeDictOffset(self.tpHandle)); if (dict != IntPtr.Zero) { Runtime.XDecref(dict); diff --git a/src/runtime/classmanager.cs b/src/runtime/classmanager.cs index 0b084a49d..837cdd16e 100644 --- a/src/runtime/classmanager.cs +++ b/src/runtime/classmanager.cs @@ -34,20 +34,14 @@ static ClassManager() dtype = typeof(MulticastDelegate); } - public static void Reset() - { - cache = new Dictionary(128); - } - /// /// Return the ClassBase-derived instance that implements a particular /// reflected managed type, creating it if it doesn't yet exist. /// internal static ClassBase GetClass(Type type) { - ClassBase cb = null; - cache.TryGetValue(type, out cb); - if (cb != null) + ClassBase cb; + if (cache.TryGetValue(type, out cb)) { return cb; } @@ -93,6 +87,11 @@ private static ClassBase CreateClass(Type type) impl = new ArrayObject(type); } + else if (type.IsKeyValuePairEnumerable()) + { + impl = new KeyValuePairEnumerableObject(type); + } + else if (type.IsInterface) { impl = new InterfaceObject(type); diff --git a/src/runtime/classobject.cs b/src/runtime/classobject.cs index 83d761fd0..ef8316c76 100644 --- a/src/runtime/classobject.cs +++ b/src/runtime/classobject.cs @@ -31,9 +31,9 @@ internal ClassObject(Type tp) : base(tp) /// internal IntPtr GetDocString() { - MethodBase[] methods = binder.GetMethods(); + var methods = binder.GetMethods(); var str = ""; - foreach (MethodBase t in methods) + foreach (var t in methods) { if (str.Length > 0) { @@ -230,10 +230,10 @@ public static int mp_ass_subscript(IntPtr ob, IntPtr idx, IntPtr v) } // Get the args passed in. - var i = Runtime.PyTuple_Size(args); + int i = Runtime.PyTuple_Size(args); IntPtr defaultArgs = cls.indexer.GetDefaultArgs(args); - var numOfDefaultArgs = Runtime.PyTuple_Size(defaultArgs); - var temp = i + numOfDefaultArgs; + int numOfDefaultArgs = Runtime.PyTuple_Size(defaultArgs); + int temp = i + numOfDefaultArgs; IntPtr real = Runtime.PyTuple_New(temp + 1); for (var n = 0; n < i; n++) { diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 502677655..a7a160aa3 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -14,11 +14,11 @@ internal CLRObject(object ob, IntPtr tp) long flags = Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Subclass) != 0) { - IntPtr dict = Marshal.ReadIntPtr(py, ObjectOffset.DictOffset(tp)); + IntPtr dict = Marshal.ReadIntPtr(py, ObjectOffset.TypeDictOffset(tp)); if (dict == IntPtr.Zero) { dict = Runtime.PyDict_New(); - Marshal.WriteIntPtr(py, ObjectOffset.DictOffset(tp), dict); + Marshal.WriteIntPtr(py, ObjectOffset.TypeDictOffset(tp), dict); } } @@ -29,9 +29,13 @@ internal CLRObject(object ob, IntPtr tp) gcHandle = gc; inst = ob; - // Fix the BaseException args (and __cause__ in case of Python 3) - // slot if wrapping a CLR exception - Exceptions.SetArgsAndCause(py); + // for performance before calling SetArgsAndCause() lets check if we are an exception + if (inst is Exception) + { + // Fix the BaseException args (and __cause__ in case of Python 3) + // slot if wrapping a CLR exception + Exceptions.SetArgsAndCause(py); + } } diff --git a/src/runtime/constructorbinding.cs b/src/runtime/constructorbinding.cs index 3908628b9..de7284031 100644 --- a/src/runtime/constructorbinding.cs +++ b/src/runtime/constructorbinding.cs @@ -119,10 +119,10 @@ public static IntPtr tp_repr(IntPtr ob) Runtime.XIncref(self.repr); return self.repr; } - MethodBase[] methods = self.ctorBinder.GetMethods(); + var methods = self.ctorBinder.GetMethods(); string name = self.type.FullName; var doc = ""; - foreach (MethodBase t in methods) + foreach (var t in methods) { if (doc.Length > 0) { diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 11c67bf82..b394a3f46 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -2,10 +2,8 @@ using System.Collections; using System.Collections.Generic; using System.Globalization; -using System.Reflection; using System.Runtime.InteropServices; using System.Security; -using System.ComponentModel; namespace Python.Runtime { @@ -31,6 +29,9 @@ private Converter() private static Type flagsType; private static Type boolType; private static Type typeType; + private static IntPtr dateTimeCtor; + private static IntPtr timeSpanCtor; + private static IntPtr tzInfoCtor; static Converter() { @@ -46,6 +47,31 @@ static Converter() flagsType = typeof(FlagsAttribute); boolType = typeof(Boolean); typeType = typeof(Type); + + IntPtr dateTimeMod = Runtime.PyImport_ImportModule("datetime"); + if (dateTimeMod == null) throw new PythonException(); + + dateTimeCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "datetime"); + if (dateTimeCtor == null) throw new PythonException(); + + timeSpanCtor = Runtime.PyObject_GetAttrString(dateTimeMod, "timedelta"); + if (timeSpanCtor == null) throw new PythonException(); + + IntPtr tzInfoMod = PythonEngine.ModuleFromString("custom_tzinfo", @" +from datetime import timedelta, tzinfo +class GMT(tzinfo): + def __init__(self, hours, minutes): + self.hours = hours + self.minutes = minutes + def utcoffset(self, dt): + return timedelta(hours=self.hours, minutes=self.minutes) + def tzname(self, dt): + return f'GMT {self.hours:00}:{self.minutes:00}' + def dst (self, dt): + return timedelta(0)").Handle; + + tzInfoCtor = Runtime.PyObject_GetAttrString(tzInfoMod, "GMT"); + if (tzInfoCtor == null) throw new PythonException(); } @@ -72,6 +98,9 @@ internal static Type GetTypeByAlias(IntPtr op) if (op == Runtime.PyBoolType) return boolType; + if (op == Runtime.PyDecimalType) + return decimalType; + return null; } @@ -101,6 +130,9 @@ internal static IntPtr GetPythonTypeByAlias(Type op) if (op == boolType) return Runtime.PyBoolType; + if (op == decimalType) + return Runtime.PyDecimalType; + return IntPtr.Zero; } @@ -135,19 +167,20 @@ internal static IntPtr ToPython(object value, Type type) return result; } - if (value is IList && !(value is INotifyPropertyChanged) && value.GetType().IsGenericType) - { - using (var resultlist = new PyList()) + var list = value as IList; + if (list != null && value.GetType().IsGenericType) + { + using (var resultList = new PyList()) { - foreach (object o in (IEnumerable)value) + for (var i = 0; i < list.Count; i++) { - using (var p = new PyObject(ToPython(o, o?.GetType()))) + using (var p = list[i].ToPython()) { - resultlist.Append(p); + resultList.Append(p); } } - Runtime.XIncref(resultlist.Handle); - return resultlist.Handle; + Runtime.XIncref(resultList.Handle); + return resultList.Handle; } } @@ -178,6 +211,17 @@ internal static IntPtr ToPython(object value, Type type) switch (tc) { case TypeCode.Object: + if (value is TimeSpan) + { + var timespan = (TimeSpan)value; + + IntPtr timeSpanArgs = Runtime.PyTuple_New(1); + Runtime.PyTuple_SetItem(timeSpanArgs, 0, Runtime.PyFloat_FromDouble(timespan.TotalDays)); + var returnTimeSpan = Runtime.PyObject_CallObject(timeSpanCtor, timeSpanArgs); + // clean up + Runtime.XDecref(timeSpanArgs); + return returnTimeSpan; + } return CLRObject.GetInstHandle(value, type); case TypeCode.String: @@ -230,6 +274,40 @@ internal static IntPtr ToPython(object value, Type type) case TypeCode.UInt64: return Runtime.PyLong_FromUnsignedLongLong((ulong)value); + case TypeCode.Decimal: + // C# decimal to python decimal has a big impact on performance + // so we will use C# double and python float + return Runtime.PyFloat_FromDouble(decimal.ToDouble((decimal) value)); + + case TypeCode.DateTime: + var datetime = (DateTime)value; + + var size = datetime.Kind == DateTimeKind.Unspecified ? 7 : 8; + + IntPtr dateTimeArgs = Runtime.PyTuple_New(size); + Runtime.PyTuple_SetItem(dateTimeArgs, 0, Runtime.PyInt_FromInt32(datetime.Year)); + Runtime.PyTuple_SetItem(dateTimeArgs, 1, Runtime.PyInt_FromInt32(datetime.Month)); + Runtime.PyTuple_SetItem(dateTimeArgs, 2, Runtime.PyInt_FromInt32(datetime.Day)); + Runtime.PyTuple_SetItem(dateTimeArgs, 3, Runtime.PyInt_FromInt32(datetime.Hour)); + Runtime.PyTuple_SetItem(dateTimeArgs, 4, Runtime.PyInt_FromInt32(datetime.Minute)); + Runtime.PyTuple_SetItem(dateTimeArgs, 5, Runtime.PyInt_FromInt32(datetime.Second)); + + // datetime.datetime 6th argument represents micro seconds + var totalSeconds = datetime.TimeOfDay.TotalSeconds; + var microSeconds = Convert.ToInt32((totalSeconds - Math.Truncate(totalSeconds)) * 1000000); + if (microSeconds == 1000000) microSeconds = 999999; + Runtime.PyTuple_SetItem(dateTimeArgs, 6, Runtime.PyInt_FromInt32(microSeconds)); + + if (size == 8) + { + Runtime.PyTuple_SetItem(dateTimeArgs, 7, TzInfo(datetime.Kind)); + } + + var returnDateTime = Runtime.PyObject_CallObject(dateTimeCtor, dateTimeArgs); + // clean up + Runtime.XDecref(dateTimeArgs); + return returnDateTime; + default: if (value is IEnumerable) { @@ -251,6 +329,18 @@ internal static IntPtr ToPython(object value, Type type) } } + private static IntPtr TzInfo(DateTimeKind kind) + { + if (kind == DateTimeKind.Unspecified) return Runtime.PyNone; + var offset = kind == DateTimeKind.Local ? DateTimeOffset.Now.Offset : TimeSpan.Zero; + IntPtr tzInfoArgs = Runtime.PyTuple_New(2); + Runtime.PyTuple_SetItem(tzInfoArgs, 0, Runtime.PyFloat_FromDouble(offset.Hours)); + Runtime.PyTuple_SetItem(tzInfoArgs, 1, Runtime.PyFloat_FromDouble(offset.Minutes)); + var returnValue = Runtime.PyObject_CallObject(tzInfoCtor, tzInfoArgs); + Runtime.XDecref(tzInfoArgs); + return returnValue; + } + /// /// In a few situations, we don't have any advisory type information @@ -294,6 +384,15 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return true; } + if(obType.IsGenericType && Runtime.PyObject_TYPE(value) == Runtime.PyListType) + { + var typeDefinition = obType.GetGenericTypeDefinition(); + if (typeDefinition == typeof(List<>)) + { + return ToList(value, obType, out result, setError); + } + } + // Common case: if the Python value is a wrapped managed object // instance, just return the wrapped object. ManagedType mt = ManagedType.GetManagedObject(value); @@ -309,6 +408,17 @@ internal static bool ToManagedValue(IntPtr value, Type obType, result = tmp; return true; } + else + { + var type = tmp.GetType(); + // check implicit conversions that receive tmp type and return obType + var conversionMethod = type.GetMethod("op_Implicit", new[] { type }); + if (conversionMethod != null && conversionMethod.ReturnType == obType) + { + result = conversionMethod.Invoke(null, new[] { tmp }); + return true; + } + } Exceptions.SetError(Exceptions.TypeError, $"value cannot be converted to {obType}"); return false; } @@ -437,6 +547,26 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return false; } + var underlyingType = Nullable.GetUnderlyingType(obType); + if (underlyingType != null) + { + return ToManagedValue(value, underlyingType, out result, setError); + } + + var opImplicit = obType.GetMethod("op_Implicit", new[] { obType }); + if (opImplicit != null) + { + if (ToManagedValue(value, opImplicit.ReturnType, out result, setError)) + { + opImplicit = obType.GetMethod("op_Implicit", new[] { result.GetType() }); + if (opImplicit != null) + { + result = opImplicit.Invoke(null, new[] { result }); + } + return opImplicit != null; + } + } + return ToPrimitive(value, obType, out result, setError); } @@ -453,6 +583,32 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo switch (tc) { + case TypeCode.Object: + if (obType == typeof(TimeSpan)) + { + op = Runtime.PyObject_Str(value); + TimeSpan ts; + var arr = Runtime.GetManagedString(op).Split(','); + string sts = arr.Length == 1 ? arr[0] : arr[1]; + if (!TimeSpan.TryParse(sts, out ts)) + { + goto type_error; + } + Runtime.XDecref(op); + + int days = 0; + if (arr.Length > 1) + { + if (!int.TryParse(arr[0].Split(' ')[0].Trim(), out days)) + { + goto type_error; + } + } + result = ts.Add(TimeSpan.FromDays(days)); + return true; + } + break; + case TypeCode.String: string st = Runtime.GetManagedString(value); if (st == null) @@ -802,6 +958,30 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo Runtime.XDecref(op); result = d; return true; + + case TypeCode.Decimal: + op = Runtime.PyObject_Str(value); + decimal m; + string sm = Runtime.GetManagedString(op); + if (!Decimal.TryParse(sm, NumberStyles.Number | NumberStyles.AllowExponent, nfi, out m)) + { + goto type_error; + } + Runtime.XDecref(op); + result = m; + return true; + + case TypeCode.DateTime: + op = Runtime.PyObject_Str(value); + DateTime dt; + string sdt = Runtime.GetManagedString(op); + if (!DateTime.TryParse(sdt, out dt)) + { + goto type_error; + } + Runtime.XDecref(op); + result = sdt.EndsWith("+00:00") ? dt.ToUniversalTime() : dt; + return true; } @@ -834,7 +1014,6 @@ private static void SetConversionError(IntPtr value, Type target) Exceptions.SetError(Exceptions.TypeError, $"Cannot convert {src} to {target}"); } - /// /// Convert a Python value to a correctly typed managed array instance. /// The Python value must support the Python sequence protocol and the @@ -843,10 +1022,10 @@ private static void SetConversionError(IntPtr value, Type target) private static bool ToArray(IntPtr value, Type obType, out object result, bool setError) { Type elementType = obType.GetElementType(); - var size = Runtime.PySequence_Size(value); + int size = Runtime.PySequence_Size(value); result = null; - if (size < 0) + if (elementType.IsGenericType) { if (setError) { @@ -857,7 +1036,49 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s Array items = Array.CreateInstance(elementType, size); - // XXX - is there a better way to unwrap this if it is a real array? + var index = 0; + result = items; + return ApplyActionToPySequence(value, obType, setError, size, elementType, o => + { + items.SetValue(o, index++); + }); + } + + /// + /// Convert a Python value to a correctly typed managed list instance. + /// The Python value must support the Python sequence protocol and the + /// items in the sequence must be convertible to the target list type. + /// + private static bool ToList(IntPtr value, Type obType, out object result, bool setError) + { + var elementType = obType.GetGenericArguments()[0]; + var size = Runtime.PySequence_Size(value); + + result = Activator.CreateInstance(obType, args: size); + var resultList = (IList)result; + return ApplyActionToPySequence(value, obType, setError, size, elementType, o => resultList.Add(o)); + } + + /// + /// Helper method that will enumerate a Python sequence convert its values to the given + /// type and send them to the provided action + /// + private static bool ApplyActionToPySequence(IntPtr value, + Type obType, + bool setError, + int size, + Type elementType, + Action action) + { + if (size < 0) + { + if (setError) + { + SetConversionError(value, obType); + } + return false; + } + for (var i = 0; i < size; i++) { object obj = null; @@ -877,15 +1098,13 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s return false; } - items.SetValue(obj, i); + action(obj); Runtime.XDecref(item); } - result = items; return true; } - /// /// Convert a Python value to a correctly typed managed enum instance. /// diff --git a/src/runtime/debughelper.cs b/src/runtime/debughelper.cs index 3fe9ee5bb..2a91a74b4 100644 --- a/src/runtime/debughelper.cs +++ b/src/runtime/debughelper.cs @@ -116,10 +116,10 @@ internal static void debug(string msg) Console.WriteLine(" {0}", msg); } - /// + /// /// Helper function to inspect/compare managed to native conversions. - /// Especially useful when debugging CustomMarshaler. - /// + /// Especially useful when debugging CustomMarshaler. + /// /// [Conditional("DEBUG")] public static void PrintHexBytes(byte[] bytes) diff --git a/src/runtime/delegatemanager.cs b/src/runtime/delegatemanager.cs index 7632816d1..bd8f1ee4c 100644 --- a/src/runtime/delegatemanager.cs +++ b/src/runtime/delegatemanager.cs @@ -181,10 +181,12 @@ A possible alternate strategy would be to create custom subclasses too "special" for this to work. It would be more work, so for now the 80/20 rule applies :) */ - public class Dispatcher + public class Dispatcher : IPyDisposable { public IntPtr target; public Type dtype; + private bool _disposed = false; + private bool _finalized = false; public Dispatcher(IntPtr target, Type dtype) { @@ -195,18 +197,25 @@ public Dispatcher(IntPtr target, Type dtype) ~Dispatcher() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; + if (_finalized || _disposed) + { + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); + } - // Note: the managed GC thread can run and try to free one of - // these *after* the Python runtime has been finalized! - if (Runtime.Py_IsInitialized() > 0) + public void Dispose() + { + if (_disposed) { - IntPtr gs = PythonEngine.AcquireLock(); - Runtime.XDecref(target); - PythonEngine.ReleaseLock(gs); + return; } + _disposed = true; + Runtime.XDecref(target); + target = IntPtr.Zero; + dtype = null; + GC.SuppressFinalize(this); } public object Dispatch(ArrayList args) @@ -267,6 +276,11 @@ public object TrueDispatch(ArrayList args) Runtime.XDecref(op); return result; } + + public IntPtr[] GetTrackedHandles() + { + return new IntPtr[] { target }; + } } diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 8bed0abfd..9023cfcfa 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -256,10 +256,7 @@ public static void SetError(Exception e) var pe = e as PythonException; if (pe != null) { - Runtime.XIncref(pe.PyType); - Runtime.XIncref(pe.PyValue); - Runtime.XIncref(pe.PyTB); - Runtime.PyErr_Restore(pe.PyType, pe.PyValue, pe.PyTB); + Runtime.PyErr_SetObject(pe.PyType, pe.PyValue); return; } @@ -279,7 +276,7 @@ public static void SetError(Exception e) /// public static bool ErrorOccurred() { - return Runtime.PyErr_Occurred() != IntPtr.Zero; + return Runtime.PyErr_Occurred() != 0; } /// diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 693a46f42..9569b0485 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -81,6 +81,27 @@ public static int tp_descr_set(IntPtr ds, IntPtr ob, IntPtr val) } + /// + /// Required Python GC support. + /// + public static int tp_traverse(IntPtr ob, IntPtr func, IntPtr args) + { + return 0; + } + + + public static int tp_clear(IntPtr ob) + { + return 0; + } + + + public static int tp_is_gc(IntPtr type) + { + return 1; + } + + /// /// Default dealloc implementation. /// diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs new file mode 100644 index 000000000..dd5c0b4dd --- /dev/null +++ b/src/runtime/finalizer.cs @@ -0,0 +1,270 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace Python.Runtime +{ + public class Finalizer + { + public class CollectArgs : EventArgs + { + public int ObjectCount { get; set; } + } + + public class ErrorArgs : EventArgs + { + public Exception Error { get; set; } + } + + public static readonly Finalizer Instance = new Finalizer(); + + public event EventHandler CollectOnce; + public event EventHandler ErrorHandler; + + public int Threshold { get; set; } + public bool Enable { get; set; } + + private ConcurrentQueue _objQueue = new ConcurrentQueue(); + private bool _pending = false; + private readonly object _collectingLock = new object(); + private Task _finalizerTask; + + #region FINALIZER_CHECK + +#if FINALIZER_CHECK + private readonly object _queueLock = new object(); + public bool RefCountValidationEnabled { get; set; } = true; +#else + public readonly bool RefCountValidationEnabled = false; +#endif + // Keep these declarations for compat even no FINALIZER_CHECK + public class IncorrectFinalizeArgs : EventArgs + { + public IntPtr Handle { get; internal set; } + public ICollection ImpactedObjects { get; internal set; } + } + + public class IncorrectRefCountException : Exception + { + public IntPtr PyPtr { get; internal set; } + private string _message; + public override string Message => _message; + + public IncorrectRefCountException(IntPtr ptr) + { + PyPtr = ptr; + IntPtr pyname = Runtime.PyObject_Unicode(PyPtr); + string name = Runtime.GetManagedString(pyname); + Runtime.XDecref(pyname); + _message = $"{name} may has a incorrect ref count"; + } + } + + public delegate bool IncorrectRefCntHandler(object sender, IncorrectFinalizeArgs e); + public event IncorrectRefCntHandler IncorrectRefCntResolver; + public bool ThrowIfUnhandleIncorrectRefCount { get; set; } = true; + + #endregion + + private Finalizer() + { + Enable = true; + Threshold = 200; + } + + public void Collect(bool forceDispose = true) + { + if (Instance._finalizerTask != null + && !Instance._finalizerTask.IsCompleted) + { + var ts = PythonEngine.BeginAllowThreads(); + Instance._finalizerTask.Wait(); + PythonEngine.EndAllowThreads(ts); + } + else if (forceDispose) + { + Instance.DisposeAll(); + } + } + + public List GetCollectedObjects() + { + return _objQueue.Select(T => new WeakReference(T)).ToList(); + } + + internal void AddFinalizedObject(IPyDisposable obj) + { + if (!Enable) + { + return; + } + if (Runtime.Py_IsInitialized() == 0) + { + // XXX: Memory will leak if a PyObject finalized after Python shutdown, + // for avoiding that case, user should call GC.Collect manual before shutdown. + return; + } +#if FINALIZER_CHECK + lock (_queueLock) +#endif + { + _objQueue.Enqueue(obj); + } + GC.ReRegisterForFinalize(obj); + if (!_pending && _objQueue.Count >= Threshold) + { + AddPendingCollect(); + } + } + + internal static void Shutdown() + { + if (Runtime.Py_IsInitialized() == 0) + { + Instance._objQueue = new ConcurrentQueue(); + return; + } + Instance.Collect(forceDispose: true); + } + + private void AddPendingCollect() + { + if(Monitor.TryEnter(_collectingLock)) + { + try + { + if (!_pending) + { + _pending = true; + // should already be complete but just in case + _finalizerTask?.Wait(); + + _finalizerTask = Task.Factory.StartNew(() => + { + using (Py.GIL()) + { + Instance.DisposeAll(); + _pending = false; + } + }); + } + } + finally + { + Monitor.Exit(_collectingLock); + } + } + } + + private void DisposeAll() + { + CollectOnce?.Invoke(this, new CollectArgs() + { + ObjectCount = _objQueue.Count + }); +#if FINALIZER_CHECK + lock (_queueLock) +#endif + { +#if FINALIZER_CHECK + ValidateRefCount(); +#endif + IPyDisposable obj; + while (_objQueue.TryDequeue(out obj)) + { + try + { + obj.Dispose(); + Runtime.CheckExceptionOccurred(); + } + catch (Exception e) + { + // We should not bother the main thread + ErrorHandler?.Invoke(this, new ErrorArgs() + { + Error = e + }); + } + } + } + } + +#if FINALIZER_CHECK + private void ValidateRefCount() + { + if (!RefCountValidationEnabled) + { + return; + } + var counter = new Dictionary(); + var holdRefs = new Dictionary(); + var indexer = new Dictionary>(); + foreach (var obj in _objQueue) + { + IntPtr[] handles = obj.GetTrackedHandles(); + foreach (var handle in handles) + { + if (handle == IntPtr.Zero) + { + continue; + } + if (!counter.ContainsKey(handle)) + { + counter[handle] = 0; + } + counter[handle]++; + if (!holdRefs.ContainsKey(handle)) + { + holdRefs[handle] = Runtime.Refcount(handle); + } + List objs; + if (!indexer.TryGetValue(handle, out objs)) + { + objs = new List(); + indexer.Add(handle, objs); + } + objs.Add(obj); + } + } + foreach (var pair in counter) + { + IntPtr handle = pair.Key; + long cnt = pair.Value; + // Tracked handle's ref count is larger than the object's holds + // it may take an unspecified behaviour if it decref in Dispose + if (cnt > holdRefs[handle]) + { + var args = new IncorrectFinalizeArgs() + { + Handle = handle, + ImpactedObjects = indexer[handle] + }; + bool handled = false; + if (IncorrectRefCntResolver != null) + { + var funcList = IncorrectRefCntResolver.GetInvocationList(); + foreach (IncorrectRefCntHandler func in funcList) + { + if (func(this, args)) + { + handled = true; + break; + } + } + } + if (!handled && ThrowIfUnhandleIncorrectRefCount) + { + throw new IncorrectRefCountException(handle); + } + } + // Make sure no other references for PyObjects after this method + indexer[handle].Clear(); + } + indexer.Clear(); + } +#endif + } +} diff --git a/src/runtime/genericutil.cs b/src/runtime/genericutil.cs index 3a230e12c..507a901c8 100644 --- a/src/runtime/genericutil.cs +++ b/src/runtime/genericutil.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Resources; namespace Python.Runtime { @@ -21,42 +20,38 @@ static GenericUtil() mapping = new Dictionary>>(); } - public static void Reset() - { - mapping = new Dictionary>>(); - } - /// /// Register a generic type that appears in a given namespace. /// internal static void Register(Type t) { - if (null == t.Namespace || null == t.Name) + lock (mapping) { - return; - } + if (null == t.Namespace || null == t.Name) + { + return; + } - Dictionary> nsmap = null; - mapping.TryGetValue(t.Namespace, out nsmap); - if (nsmap == null) - { - nsmap = new Dictionary>(); - mapping[t.Namespace] = nsmap; - } - string basename = t.Name; - int tick = basename.IndexOf("`"); - if (tick > -1) - { - basename = basename.Substring(0, tick); - } - List gnames = null; - nsmap.TryGetValue(basename, out gnames); - if (gnames == null) - { - gnames = new List(); - nsmap[basename] = gnames; + Dictionary> nsmap; + if (!mapping.TryGetValue(t.Namespace, out nsmap)) + { + nsmap = new Dictionary>(); + mapping[t.Namespace] = nsmap; + } + string basename = t.Name; + int tick = basename.IndexOf("`"); + if (tick > -1) + { + basename = basename.Substring(0, tick); + } + List gnames; + if (!nsmap.TryGetValue(basename, out gnames)) + { + gnames = new List(); + nsmap[basename] = gnames; + } + gnames.Add(t.Name); } - gnames.Add(t.Name); } /// @@ -64,18 +59,20 @@ internal static void Register(Type t) /// public static List GetGenericBaseNames(string ns) { - Dictionary> nsmap = null; - mapping.TryGetValue(ns, out nsmap); - if (nsmap == null) - { - return null; - } - var names = new List(); - foreach (string key in nsmap.Keys) + lock (mapping) { - names.Add(key); + Dictionary> nsmap; + if (!mapping.TryGetValue(ns, out nsmap)) + { + return null; + } + var names = new List(); + foreach (string key in nsmap.Keys) + { + names.Add(key); + } + return names; } - return names; } /// @@ -105,38 +102,38 @@ public static List GenericsForType(Type t) public static List GenericsByName(string ns, string basename) { - Dictionary> nsmap = null; - mapping.TryGetValue(ns, out nsmap); - if (nsmap == null) + lock (mapping) { - return null; - } + Dictionary> nsmap; + if (!mapping.TryGetValue(ns, out nsmap)) + { + return null; + } - int tick = basename.IndexOf("`"); - if (tick > -1) - { - basename = basename.Substring(0, tick); - } + int tick = basename.IndexOf("`"); + if (tick > -1) + { + basename = basename.Substring(0, tick); + } - List names = null; - nsmap.TryGetValue(basename, out names); - if (names == null) - { - return null; - } + List names; + if (!nsmap.TryGetValue(basename, out names)) + { + return null; + } - var result = new List(); - foreach (string name in names) - { - string qname = ns + "." + name; - Type o = AssemblyManager.LookupType(qname); - if (o != null) + var result = new List(); + foreach (string name in names) { - result.Add(o); + string qname = ns + "." + name; + Type o = AssemblyManager.LookupType(qname); + if (o != null) + { + result.Add(o); + } } + return result; } - - return result; } /// @@ -144,17 +141,19 @@ public static List GenericsByName(string ns, string basename) /// public static string GenericNameForBaseName(string ns, string name) { - Dictionary> nsmap = null; - mapping.TryGetValue(ns, out nsmap); - if (nsmap == null) - { - return null; - } - List gnames = null; - nsmap.TryGetValue(name, out gnames); - if (gnames?.Count > 0) + lock (mapping) { - return gnames[0]; + Dictionary> nsmap; + if (!mapping.TryGetValue(ns, out nsmap)) + { + return null; + } + List gnames = null; + nsmap.TryGetValue(name, out gnames); + if (gnames?.Count > 0) + { + return gnames[0]; + } } return null; } diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 7e4a208f5..bc9ac5eee 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -155,7 +155,7 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) // hook is saved as this.py_import. This version handles CLR // import and defers to the normal builtin for everything else. - var num_args = Runtime.PyTuple_Size(args); + int num_args = Runtime.PyTuple_Size(args); if (num_args < 1) { return Exceptions.RaiseTypeError("__import__() takes at least 1 argument (0 given)"); @@ -237,11 +237,6 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) if (res != IntPtr.Zero) { // There was no error. - if (fromlist && IsLoadAll(fromList)) - { - var mod = ManagedType.GetManagedObject(res) as ModuleObject; - mod?.LoadNames(); - } return res; } // There was an error @@ -295,11 +290,6 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) { if (fromlist) { - if (IsLoadAll(fromList)) - { - var mod = ManagedType.GetManagedObject(module) as ModuleObject; - mod?.LoadNames(); - } Runtime.XIncref(module); return module; } @@ -355,33 +345,20 @@ public static IntPtr __import__(IntPtr self, IntPtr args, IntPtr kw) } } - { - var mod = fromlist ? tail : head; + ModuleObject mod = fromlist ? tail : head; - if (fromlist && IsLoadAll(fromList)) + if (fromlist && Runtime.PySequence_Size(fromList) == 1) + { + IntPtr fp = Runtime.PySequence_GetItem(fromList, 0); + if (!CLRModule.preload && Runtime.GetManagedString(fp) == "*") { mod.LoadNames(); } - - Runtime.XIncref(mod.pyHandle); - return mod.pyHandle; + Runtime.XDecref(fp); } - } - private static bool IsLoadAll(IntPtr fromList) - { - if (CLRModule.preload) - { - return false; - } - if (Runtime.PySequence_Size(fromList) != 1) - { - return false; - } - IntPtr fp = Runtime.PySequence_GetItem(fromList, 0); - bool res = Runtime.GetManagedString(fp) == "*"; - Runtime.XDecref(fp); - return res; + Runtime.XIncref(mod.pyHandle); + return mod.pyHandle; } } } diff --git a/src/runtime/indexer.cs b/src/runtime/indexer.cs index 71f7e7aa1..6379ba968 100644 --- a/src/runtime/indexer.cs +++ b/src/runtime/indexer.cs @@ -56,15 +56,14 @@ internal void SetItem(IntPtr inst, IntPtr args) internal bool NeedsDefaultArgs(IntPtr args) { - var pynargs = Runtime.PyTuple_Size(args); - MethodBase[] methods = SetterBinder.GetMethods(); - if (methods.Length == 0) + var methods = SetterBinder.GetMethods(); + if (methods.Count == 0) { return false; } + int pynargs = Runtime.PyTuple_Size(args); - MethodBase mi = methods[0]; - ParameterInfo[] pi = mi.GetParameters(); + var pi = methods[0].ParameterInfo; // need to subtract one for the value int clrnargs = pi.Length - 1; if (pynargs == clrnargs || pynargs > clrnargs) @@ -72,7 +71,7 @@ internal bool NeedsDefaultArgs(IntPtr args) return false; } - for (var v = pynargs; v < clrnargs; v++) + for (int v = pynargs; v < clrnargs; v++) { if (pi[v].DefaultValue == DBNull.Value) { @@ -95,12 +94,11 @@ internal IntPtr GetDefaultArgs(IntPtr args) { return Runtime.PyTuple_New(0); } - var pynargs = Runtime.PyTuple_Size(args); + int pynargs = Runtime.PyTuple_Size(args); // Get the default arg tuple - MethodBase[] methods = SetterBinder.GetMethods(); - MethodBase mi = methods[0]; - ParameterInfo[] pi = mi.GetParameters(); + var methods = SetterBinder.GetMethods(); + ParameterInfo[] pi = methods[0].ParameterInfo; int clrnargs = pi.Length - 1; IntPtr defaultArgs = Runtime.PyTuple_New(clrnargs - pynargs); for (var i = 0; i < clrnargs - pynargs; i++) diff --git a/src/runtime/interfaceobject.cs b/src/runtime/interfaceobject.cs index 616ced6bd..ce1bc9eb0 100644 --- a/src/runtime/interfaceobject.cs +++ b/src/runtime/interfaceobject.cs @@ -36,7 +36,7 @@ static InterfaceObject() public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { var self = (InterfaceObject)GetManagedObject(tp); - var nargs = Runtime.PyTuple_Size(args); + int nargs = Runtime.PyTuple_Size(args); Type type = self.type; object obj; diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 4ae4b61e0..9b4ab7f57 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -1,6 +1,7 @@ using System; using System.Collections; -using System.Collections.Specialized; +using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.InteropServices; using System.Reflection; using System.Text; @@ -67,11 +68,47 @@ public ModulePropertyAttribute() } } + internal static class ManagedDataOffsets + { + static ManagedDataOffsets() + { + FieldInfo[] fi = typeof(ManagedDataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); + for (int i = 0; i < fi.Length; i++) + { + fi[i].SetValue(null, -(i * IntPtr.Size) - IntPtr.Size); + } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class ObjectOffset + size = fi.Length * IntPtr.Size; + } + + public static readonly int ob_data; + public static readonly int ob_dict; + + private static int BaseOffset(IntPtr type) + { + Debug.Assert(type != IntPtr.Zero); + int typeSize = Marshal.ReadInt32(type, TypeOffset.tp_basicsize); + Debug.Assert(typeSize > 0 && typeSize <= ExceptionOffset.Size()); + return typeSize; + } + public static int DataOffset(IntPtr type) + { + return BaseOffset(type) + ob_data; + } + + public static int DictOffset(IntPtr type) + { + return BaseOffset(type) + ob_dict; + } + + public static int Size { get { return size; } } + + private static readonly int size; + } + + internal static class OriginalObjectOffsets { - static ObjectOffset() + static OriginalObjectOffsets() { int size = IntPtr.Size; var n = 0; // Py_TRACE_REFS add two pointers to PyObject_HEAD @@ -82,42 +119,58 @@ static ObjectOffset() #endif ob_refcnt = (n + 0) * size; ob_type = (n + 1) * size; - ob_dict = (n + 2) * size; - ob_data = (n + 3) * size; } - public static int magic(IntPtr ob) + public static int Size { get { return size; } } + + private static readonly int size = +#if PYTHON_WITH_PYDEBUG + 4 * IntPtr.Size; +#else + 2 * IntPtr.Size; +#endif + +#if PYTHON_WITH_PYDEBUG + public static int _ob_next; + public static int _ob_prev; +#endif + public static int ob_refcnt; + public static int ob_type; + } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal class ObjectOffset + { + static ObjectOffset() { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) - { - return ExceptionOffset.ob_data; - } - return ob_data; +#if PYTHON_WITH_PYDEBUG + _ob_next = OriginalObjectOffsets._ob_next; + _ob_prev = OriginalObjectOffsets._ob_prev; +#endif + ob_refcnt = OriginalObjectOffsets.ob_refcnt; + ob_type = OriginalObjectOffsets.ob_type; + + size = OriginalObjectOffsets.Size + ManagedDataOffsets.Size; } - public static int DictOffset(IntPtr ob) + public static int magic(IntPtr type) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) - { - return ExceptionOffset.ob_dict; - } - return ob_dict; + return ManagedDataOffsets.DataOffset(type); } - public static int Size(IntPtr ob) + public static int TypeDictOffset(IntPtr type) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) + return ManagedDataOffsets.DictOffset(type); + } + + public static int Size(IntPtr pyType) + { + if (IsException(pyType)) { return ExceptionOffset.Size(); } -#if PYTHON_WITH_PYDEBUG - return 6 * IntPtr.Size; -#else - return 4 * IntPtr.Size; -#endif + + return size; } #if PYTHON_WITH_PYDEBUG @@ -126,8 +179,15 @@ public static int Size(IntPtr ob) #endif public static int ob_refcnt; public static int ob_type; - private static int ob_dict; - private static int ob_data; + private static readonly int size; + + private static bool IsException(IntPtr pyObject) + { + var type = Runtime.PyObject_TYPE(pyObject); + return Runtime.PyObjectType_TypeCheck(type, Exceptions.BaseException) + || Runtime.PyObjectType_TypeCheck(type, Runtime.PyTypeType) + && Runtime.PyType_IsSubtype(pyObject, Exceptions.BaseException); + } } [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] @@ -136,19 +196,17 @@ internal class ExceptionOffset static ExceptionOffset() { Type type = typeof(ExceptionOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; + FieldInfo[] fi = type.GetFields(BindingFlags.Static | BindingFlags.Public); for (int i = 0; i < fi.Length; i++) { - fi[i].SetValue(null, (i * size) + ObjectOffset.ob_type + size); + fi[i].SetValue(null, (i * IntPtr.Size) + OriginalObjectOffsets.Size); } - } - public static int Size() - { - return ob_data + IntPtr.Size; + size = fi.Length * IntPtr.Size + OriginalObjectOffsets.Size + ManagedDataOffsets.Size; } + public static int Size() { return size; } + // PyException_HEAD // (start after PyObject_HEAD) public static int dict = 0; @@ -162,9 +220,7 @@ public static int Size() public static int suppress_context = 0; #endif - // extra c# data - public static int ob_dict; - public static int ob_data; + private static readonly int size; } diff --git a/src/runtime/interop37.cs b/src/runtime/interop37.cs deleted file mode 100644 index d5fc76ad3..000000000 --- a/src/runtime/interop37.cs +++ /dev/null @@ -1,149 +0,0 @@ -// Auto-generated by geninterop.py. -// DO NOT MODIFIY BY HAND. - - -#if PYTHON37 -using System; -using System.Collections; -using System.Collections.Specialized; -using System.Runtime.InteropServices; -using System.Reflection; -using System.Text; - -namespace Python.Runtime -{ - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class TypeOffset - { - static TypeOffset() - { - Type type = typeof(TypeOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; - for (int i = 0; i < fi.Length; i++) - { - fi[i].SetValue(null, i * size); - } - } - - public static int magic() - { - return ob_size; - } - - // Auto-generated from PyHeapTypeObject in Python.h - public static int ob_refcnt = 0; - public static int ob_type = 0; - public static int ob_size = 0; - public static int tp_name = 0; - public static int tp_basicsize = 0; - public static int tp_itemsize = 0; - public static int tp_dealloc = 0; - public static int tp_print = 0; - public static int tp_getattr = 0; - public static int tp_setattr = 0; - public static int tp_as_async = 0; - public static int tp_repr = 0; - public static int tp_as_number = 0; - public static int tp_as_sequence = 0; - public static int tp_as_mapping = 0; - public static int tp_hash = 0; - public static int tp_call = 0; - public static int tp_str = 0; - public static int tp_getattro = 0; - public static int tp_setattro = 0; - public static int tp_as_buffer = 0; - public static int tp_flags = 0; - public static int tp_doc = 0; - public static int tp_traverse = 0; - public static int tp_clear = 0; - public static int tp_richcompare = 0; - public static int tp_weaklistoffset = 0; - public static int tp_iter = 0; - public static int tp_iternext = 0; - public static int tp_methods = 0; - public static int tp_members = 0; - public static int tp_getset = 0; - public static int tp_base = 0; - public static int tp_dict = 0; - public static int tp_descr_get = 0; - public static int tp_descr_set = 0; - public static int tp_dictoffset = 0; - public static int tp_init = 0; - public static int tp_alloc = 0; - public static int tp_new = 0; - public static int tp_free = 0; - public static int tp_is_gc = 0; - public static int tp_bases = 0; - public static int tp_mro = 0; - public static int tp_cache = 0; - public static int tp_subclasses = 0; - public static int tp_weaklist = 0; - public static int tp_del = 0; - public static int tp_version_tag = 0; - public static int tp_finalize = 0; - public static int am_await = 0; - public static int am_aiter = 0; - public static int am_anext = 0; - public static int nb_add = 0; - public static int nb_subtract = 0; - public static int nb_multiply = 0; - public static int nb_remainder = 0; - public static int nb_divmod = 0; - public static int nb_power = 0; - public static int nb_negative = 0; - public static int nb_positive = 0; - public static int nb_absolute = 0; - public static int nb_bool = 0; - public static int nb_invert = 0; - public static int nb_lshift = 0; - public static int nb_rshift = 0; - public static int nb_and = 0; - public static int nb_xor = 0; - public static int nb_or = 0; - public static int nb_int = 0; - public static int nb_reserved = 0; - public static int nb_float = 0; - public static int nb_inplace_add = 0; - public static int nb_inplace_subtract = 0; - public static int nb_inplace_multiply = 0; - public static int nb_inplace_remainder = 0; - public static int nb_inplace_power = 0; - public static int nb_inplace_lshift = 0; - public static int nb_inplace_rshift = 0; - public static int nb_inplace_and = 0; - public static int nb_inplace_xor = 0; - public static int nb_inplace_or = 0; - public static int nb_floor_divide = 0; - public static int nb_true_divide = 0; - public static int nb_inplace_floor_divide = 0; - public static int nb_inplace_true_divide = 0; - public static int nb_index = 0; - public static int nb_matrix_multiply = 0; - public static int nb_inplace_matrix_multiply = 0; - public static int mp_length = 0; - public static int mp_subscript = 0; - public static int mp_ass_subscript = 0; - public static int sq_length = 0; - public static int sq_concat = 0; - public static int sq_repeat = 0; - public static int sq_item = 0; - public static int was_sq_slice = 0; - public static int sq_ass_item = 0; - public static int was_sq_ass_slice = 0; - public static int sq_contains = 0; - public static int sq_inplace_concat = 0; - public static int sq_inplace_repeat = 0; - public static int bf_getbuffer = 0; - public static int bf_releasebuffer = 0; - public static int name = 0; - public static int ht_slots = 0; - public static int qualname = 0; - public static int ht_cached_keys = 0; - - /* here are optional user slots, followed by the members. */ - public static int members = 0; - } -} - -#endif diff --git a/src/runtime/keyvaluepairenumerableobject.cs b/src/runtime/keyvaluepairenumerableobject.cs new file mode 100644 index 000000000..c1644442c --- /dev/null +++ b/src/runtime/keyvaluepairenumerableobject.cs @@ -0,0 +1,112 @@ +using System; +using System.Collections.Generic; +using System.Reflection; + +namespace Python.Runtime +{ + /// + /// Implements a Python type for managed KeyValuePairEnumerable (dictionaries). + /// This type is essentially the same as a ClassObject, except that it provides + /// sequence semantics to support natural dictionary usage (__contains__ and __len__) + /// from Python. + /// + internal class KeyValuePairEnumerableObject : ClassObject + { + private static Dictionary, MethodInfo> methodsByType = new Dictionary, MethodInfo>(); + private static List requiredMethods = new List { "Count", "ContainsKey" }; + + internal static bool VerifyMethodRequirements(Type type) + { + foreach (var requiredMethod in requiredMethods) + { + var method = type.GetMethod(requiredMethod); + if (method == null) + { + method = type.GetMethod($"get_{requiredMethod}"); + if (method == null) + { + return false; + } + } + + var key = Tuple.Create(type, requiredMethod); + methodsByType.Add(key, method); + } + + return true; + } + + internal KeyValuePairEnumerableObject(Type tp) : base(tp) + { + + } + + internal override bool CanSubclass() => false; + + /// + /// Implements __len__ for dictionary types. + /// + public static int mp_length(IntPtr ob) + { + var obj = (CLRObject)GetManagedObject(ob); + var self = obj.inst; + + var key = Tuple.Create(self.GetType(), "Count"); + var methodInfo = methodsByType[key]; + + return (int)methodInfo.Invoke(self, null); + } + + /// + /// Implements __contains__ for dictionary types. + /// + public static int sq_contains(IntPtr ob, IntPtr v) + { + var obj = (CLRObject)GetManagedObject(ob); + var self = obj.inst; + + var key = Tuple.Create(self.GetType(), "ContainsKey"); + var methodInfo = methodsByType[key]; + + var parameters = methodInfo.GetParameters(); + object arg; + if (!Converter.ToManaged(v, parameters[0].ParameterType, out arg, false)) + { + Exceptions.SetError(Exceptions.TypeError, + $"invalid parameter type for sq_contains: should be {Converter.GetTypeByAlias(v)}, found {parameters[0].ParameterType}"); + } + + return (bool)methodInfo.Invoke(self, new[] { arg }) ? 1 : 0; + } + } + + public static class KeyValuePairEnumerableObjectExtension + { + public static bool IsKeyValuePairEnumerable(this Type type) + { + var iEnumerableType = typeof(IEnumerable<>); + var keyValuePairType = typeof(KeyValuePair<,>); + + var interfaces = type.GetInterfaces(); + foreach (var i in interfaces) + { + if (i.IsGenericType && + i.GetGenericTypeDefinition() == iEnumerableType) + { + var arguments = i.GetGenericArguments(); + if (arguments.Length != 1) continue; + + var a = arguments[0]; + if (a.IsGenericType && + a.GetGenericTypeDefinition() == keyValuePairType && + a.GetGenericArguments().Length == 2) + { + return KeyValuePairEnumerableObject.VerifyMethodRequirements(type); + } + } + } + + return false; + } + } +} diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 3191da949..23f5898d1 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -33,7 +33,7 @@ internal static ManagedType GetManagedObject(IntPtr ob) { IntPtr op = tp == ob ? Marshal.ReadIntPtr(tp, TypeOffset.magic()) - : Marshal.ReadIntPtr(ob, ObjectOffset.magic(ob)); + : Marshal.ReadIntPtr(ob, ObjectOffset.magic(tp)); if (op == IntPtr.Zero) { return null; diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 8853c2d5e..3295ab110 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -29,7 +29,7 @@ public static IntPtr Initialize() /// public static IntPtr tp_new(IntPtr tp, IntPtr args, IntPtr kw) { - var len = Runtime.PyTuple_Size(args); + int len = Runtime.PyTuple_Size(args); if (len < 3) { return Exceptions.RaiseTypeError("invalid argument list"); diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 5e800c36f..854f386ab 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -1,5 +1,6 @@ using System; using System.Collections; +using System.Collections.Generic; using System.Reflection; namespace Python.Runtime @@ -12,19 +13,18 @@ namespace Python.Runtime /// internal class MethodBinder { - public ArrayList list; - public MethodBase[] methods; + private List list; public bool init = false; public bool allow_threads = true; internal MethodBinder() { - list = new ArrayList(); + list = new List(); } internal MethodBinder(MethodInfo mi) { - list = new ArrayList { mi }; + list = new List { new MethodInformation(mi, mi.GetParameters()) }; } public int Count @@ -34,7 +34,9 @@ public int Count internal void AddMethod(MethodBase m) { - list.Add(m); + // we added a new method so we have to re sort the method list + init = false; + list.Add(new MethodInformation(m, m.GetParameters())); } /// @@ -154,16 +156,15 @@ internal static MethodInfo MatchSignatureAndParameters(MethodInfo[] mi, Type[] g /// is arranged in order of precedence (done lazily to avoid doing it /// at all for methods that are never called). /// - internal MethodBase[] GetMethods() + internal List GetMethods() { if (!init) { // I'm sure this could be made more efficient. list.Sort(new MethodSorter()); - methods = (MethodBase[])list.ToArray(typeof(MethodBase)); init = true; } - return methods; + return list; } /// @@ -174,9 +175,10 @@ internal MethodBase[] GetMethods() /// Based from Jython `org.python.core.ReflectedArgs.precedence` /// See: https://github.com/jythontools/jython/blob/master/src/org/python/core/ReflectedArgs.java#L192 /// - internal static int GetPrecedence(MethodBase mi) + private static int GetPrecedence(MethodInformation methodInformation) { - ParameterInfo[] pi = mi.GetParameters(); + ParameterInfo[] pi = methodInformation.ParameterInfo; + var mi = methodInformation.MethodBase; int val = mi.IsStatic ? 3000 : 0; int num = pi.Length; @@ -186,6 +188,13 @@ internal static int GetPrecedence(MethodBase mi) val += ArgPrecedence(pi[i].ParameterType); } + var info = mi as MethodInfo; + if (info != null) + { + val += ArgPrecedence(info.ReturnType); + val += mi.DeclaringType == mi.ReflectedType ? 0 : 3000; + } + return val; } @@ -200,6 +209,11 @@ internal static int ArgPrecedence(Type t) return 3000; } + if (t.IsAssignableFrom(typeof(PyObject))) + { + return -1; + } + TypeCode tc = Type.GetTypeCode(t); // TODO: Clean up switch (tc) @@ -278,67 +292,40 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, MethodInfo[] methodinfo) { // loop to find match, return invoker w/ or /wo error - MethodBase[] _methods = null; - var pynargs = (int)Runtime.PyTuple_Size(args); + int pynargs = Runtime.PyTuple_Size(args); object arg; var isGeneric = false; - ArrayList defaultArgList = null; - if (info != null) - { - _methods = new MethodBase[1]; - _methods.SetValue(info, 0); - } - else - { - _methods = GetMethods(); - } + ArrayList defaultArgList; Type clrtype; + Binding bindingUsingImplicitConversion = null; + + var methods = info == null ? GetMethods() + : new List(1) { new MethodInformation(info, info.GetParameters()) }; + // TODO: Clean up - foreach (MethodBase mi in _methods) + foreach (var methodInformation in methods) { + var mi = methodInformation.MethodBase; + var pi = methodInformation.ParameterInfo; + if (mi.IsGenericMethod) { isGeneric = true; } - ParameterInfo[] pi = mi.GetParameters(); - var clrnargs = pi.Length; - var match = false; - var arrayStart = -1; - var outs = 0; - - if (pynargs == clrnargs) - { - match = true; - } - else if (pynargs < clrnargs) - { - match = true; - defaultArgList = new ArrayList(); - for (var v = pynargs; v < clrnargs; v++) - { - if (pi[v].DefaultValue == DBNull.Value) - { - match = false; - } - else - { - defaultArgList.Add(pi[v].DefaultValue); - } - } - } - else if (pynargs > clrnargs && clrnargs > 0 && - Attribute.IsDefined(pi[clrnargs - 1], typeof(ParamArrayAttribute))) - { - // This is a `foo(params object[] bar)` style method - match = true; - arrayStart = clrnargs - 1; - } - - if (match) + int clrnargs = pi.Length; + int arrayStart; + + if (CheckMethodArgumentsMatch(clrnargs, + pynargs, + pi, + out arrayStart, + out defaultArgList)) { + var outs = 0; var margs = new object[clrnargs]; + var usedImplicitConversion = false; - for (int n = 0; n < clrnargs; n++) + for (var n = 0; n < clrnargs; n++) { IntPtr op; if (n < pynargs) @@ -359,7 +346,7 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth // is necessary clrtype = null; IntPtr pyoptype; - if (_methods.Length > 1) + if (methods.Count > 1) { pyoptype = IntPtr.Zero; pyoptype = Runtime.PyObject_Type(op); @@ -394,14 +381,33 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth } if (!typematch) { + // this takes care of nullables + var underlyingType = Nullable.GetUnderlyingType(pi[n].ParameterType); + if (underlyingType == null) + { + underlyingType = pi[n].ParameterType; + } // this takes care of enum values - TypeCode argtypecode = Type.GetTypeCode(pi[n].ParameterType); + TypeCode argtypecode = Type.GetTypeCode(underlyingType); TypeCode paramtypecode = Type.GetTypeCode(clrtype); if (argtypecode == paramtypecode) { typematch = true; clrtype = pi[n].ParameterType; } + // accepts non-decimal numbers in decimal parameters + if (underlyingType == typeof(decimal)) + { + clrtype = pi[n].ParameterType; + typematch = Converter.ToManaged(op, clrtype, out arg, false); + } + // this takes care of implicit conversions + var opImplicit = pi[n].ParameterType.GetMethod("op_Implicit", new[] { clrtype }); + if (opImplicit != null) + { + usedImplicitConversion = typematch = opImplicit.ReturnType == pi[n].ParameterType; + clrtype = pi[n].ParameterType; + } } Runtime.XDecref(pyoptype); if (!typematch) @@ -473,9 +479,31 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth target = co.inst; } - return new Binding(mi, target, margs, outs); + var binding = new Binding(mi, target, margs, outs); + if (usedImplicitConversion) + { + // lets just keep the first binding using implicit conversion + // this is to respect method order/precedence + if (bindingUsingImplicitConversion == null) + { + // in this case we will not return the binding yet in case there is a match + // which does not use implicit conversions, which will return directly + bindingUsingImplicitConversion = binding; + } + } + else + { + return binding; + } } } + + // if we generated a binding using implicit conversion return it + if (bindingUsingImplicitConversion != null) + { + return bindingUsingImplicitConversion; + } + // We weren't able to find a matching method but at least one // is a generic method and info is null. That happens when a generic // method was not called using the [] syntax. Let's introspect the @@ -489,6 +517,51 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth return null; } + /// + /// This helper method will perform an initial check to determine if we found a matching + /// method based on its parameters count and type + /// + private bool CheckMethodArgumentsMatch(int clrnargs, + int pynargs, + ParameterInfo[] parameterInfo, + out int arrayStart, + out ArrayList defaultArgList) + { + arrayStart = -1; + defaultArgList = null; + + var match = false; + if (pynargs == clrnargs) + { + match = true; + } + else if (pynargs < clrnargs) + { + match = true; + defaultArgList = new ArrayList(); + for (var v = pynargs; v < clrnargs && match; v++) + { + if (parameterInfo[v].DefaultValue == DBNull.Value) + { + match = false; + } + else + { + defaultArgList.Add(parameterInfo[v].DefaultValue); + } + } + } + else if (pynargs > clrnargs && clrnargs > 0 && + Attribute.IsDefined(parameterInfo[clrnargs - 1], typeof(ParamArrayAttribute))) + { + // This is a `foo(params object[] bar)` style method + match = true; + arrayStart = clrnargs - 1; + } + + return match; + } + internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw) { return Invoke(inst, args, kw, null, null); @@ -591,40 +664,47 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i return Converter.ToPython(result, mi.ReturnType); } - } - - /// - /// Utility class to sort method info by parameter type precedence. - /// - internal class MethodSorter : IComparer - { - int IComparer.Compare(object m1, object m2) + /// + /// Utility class to store the information about a + /// + internal class MethodInformation { - var me1 = (MethodBase)m1; - var me2 = (MethodBase)m2; - if (me1.DeclaringType != me2.DeclaringType) - { - // m2's type derives from m1's type, favor m2 - if (me1.DeclaringType.IsAssignableFrom(me2.DeclaringType)) - return 1; + public MethodBase MethodBase { get; } - // m1's type derives from m2's type, favor m1 - if (me2.DeclaringType.IsAssignableFrom(me1.DeclaringType)) - return -1; + public ParameterInfo[] ParameterInfo { get; } + + public MethodInformation(MethodBase methodBase, ParameterInfo[] parameterInfo) + { + MethodBase = methodBase; + ParameterInfo = parameterInfo; } - int p1 = MethodBinder.GetPrecedence((MethodBase)m1); - int p2 = MethodBinder.GetPrecedence((MethodBase)m2); - if (p1 < p2) + public override string ToString() { - return -1; + return MethodBase.ToString(); } - if (p1 > p2) + } + + /// + /// Utility class to sort method info by parameter type precedence. + /// + private class MethodSorter : IComparer + { + public int Compare(MethodInformation x, MethodInformation y) { - return 1; + int p1 = GetPrecedence(x); + int p2 = GetPrecedence(y); + if (p1 < p2) + { + return -1; + } + if (p1 > p2) + { + return 1; + } + return 0; } - return 0; } } diff --git a/src/runtime/methodbinding.cs b/src/runtime/methodbinding.cs index f402f91f8..3985ed2cb 100644 --- a/src/runtime/methodbinding.cs +++ b/src/runtime/methodbinding.cs @@ -104,7 +104,7 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) { if (self.info.IsGenericMethod) { - var len = Runtime.PyTuple_Size(args); //FIXME: Never used + int len = Runtime.PyTuple_Size(args); //FIXME: Never used Type[] sigTp = Runtime.PythonArgsToTypeArray(args, true); if (sigTp != null) { @@ -129,7 +129,7 @@ public static IntPtr tp_call(IntPtr ob, IntPtr args, IntPtr kw) if (target == IntPtr.Zero && !self.m.IsStatic()) { - var len = Runtime.PyTuple_Size(args); + int len = Runtime.PyTuple_Size(args); if (len < 1) { Exceptions.SetError(Exceptions.TypeError, "not enough arguments"); diff --git a/src/runtime/methodobject.cs b/src/runtime/methodobject.cs index 8df9c8029..2e67787d2 100644 --- a/src/runtime/methodobject.cs +++ b/src/runtime/methodobject.cs @@ -68,14 +68,14 @@ internal IntPtr GetDocString() } var str = ""; Type marker = typeof(DocStringAttribute); - MethodBase[] methods = binder.GetMethods(); - foreach (MethodBase method in methods) + var methods = binder.GetMethods(); + foreach (var method in methods) { if (str.Length > 0) { str += Environment.NewLine; } - var attrs = (Attribute[])method.GetCustomAttributes(marker, false); + var attrs = (Attribute[])method.MethodBase.GetCustomAttributes(marker, false); if (attrs.Length == 0) { str += method.ToString(); diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 8af722d29..4522fb9fa 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -53,7 +53,7 @@ public ModuleObject(string name) Runtime.XDecref(pyfilename); Runtime.XDecref(pydocstring); - Marshal.WriteIntPtr(pyHandle, ObjectOffset.DictOffset(pyHandle), dict); + Marshal.WriteIntPtr(pyHandle, ObjectOffset.TypeDictOffset(tpHandle), dict); InitializeModuleMembers(); } @@ -67,9 +67,8 @@ public ModuleObject(string name) /// public ManagedType GetAttribute(string name, bool guess) { - ManagedType cached = null; - cache.TryGetValue(name, out cached); - if (cached != null) + ManagedType cached; + if (cache.TryGetValue(name, out cached)) { return cached; } @@ -78,16 +77,6 @@ public ManagedType GetAttribute(string name, bool guess) ClassBase c; Type type; - //if (AssemblyManager.IsValidNamespace(name)) - //{ - // IntPtr py_mod_name = Runtime.PyString_FromString(name); - // IntPtr modules = Runtime.PyImport_GetModuleDict(); - // IntPtr module = Runtime.PyDict_GetItem(modules, py_mod_name); - // if (module != IntPtr.Zero) - // return (ManagedType)this; - // return null; - //} - string qname = _namespace == string.Empty ? name : _namespace + "." + name; @@ -186,21 +175,9 @@ private void StoreAttribute(string name, ManagedType ob) /// public void LoadNames() { - ManagedType m = null; - foreach (string name in AssemblyManager.GetNames(_namespace)) + foreach (var name in AssemblyManager.GetNames(_namespace)) { - cache.TryGetValue(name, out m); - if (m != null) - { - continue; - } - IntPtr attr = Runtime.PyDict_GetItemString(dict, name); - // If __dict__ has already set a custom property, skip it. - if (attr != IntPtr.Zero) - { - continue; - } - GetAttribute(name, true); + var attr = GetAttribute(name, true); } } @@ -335,17 +312,6 @@ public CLRModule() : base("clr") } } - public static void Reset() - { - hacked = false; - interactive_preload = true; - preload = false; - - // XXX Test performance of new features // - _SuppressDocs = false; - _SuppressOverloads = false; - } - /// /// The initializing of the preload hook has to happen as late as /// possible since sys.ps1 is created after the CLR module is diff --git a/src/runtime/propertyobject.cs b/src/runtime/propertyobject.cs index f2c97f163..cc6125157 100644 --- a/src/runtime/propertyobject.cs +++ b/src/runtime/propertyobject.cs @@ -1,4 +1,5 @@ using System; +using System.Linq.Expressions; using System.Reflection; using System.Security.Permissions; @@ -12,6 +13,10 @@ internal class PropertyObject : ExtensionType private PropertyInfo info; private MethodInfo getter; private MethodInfo setter; + private bool getterCacheFailed; + private bool setterCacheFailed; + private Func getterCache; + private Action setterCache; [StrongNameIdentityPermission(SecurityAction.Assert)] public PropertyObject(PropertyInfo md) @@ -67,7 +72,21 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) try { - result = self.info.GetValue(co.inst, null); + if (self.getterCache == null && !self.getterCacheFailed) + { + // if the getter is not public 'GetGetMethod' will not find it + // but calling 'GetValue' will work, so for backwards compatibility + // we will use it instead + self.getterCache = BuildGetter(self.info); + if (self.getterCache == null) + { + self.getterCacheFailed = true; + } + } + + result = self.getterCacheFailed ? + self.info.GetValue(co.inst, null) + : self.getterCache(co.inst); return Converter.ToPython(result, self.info.PropertyType); } catch (Exception e) @@ -81,7 +100,6 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) } } - /// /// Descriptor __set__ implementation. This method sets the value of /// a property based on the given Python value. The Python value must @@ -132,7 +150,27 @@ public static IntPtr tp_descr_get(IntPtr ds, IntPtr ob, IntPtr tp) Exceptions.RaiseTypeError("invalid target"); return -1; } - self.info.SetValue(co.inst, newval, null); + + if (self.setterCache == null && !self.setterCacheFailed) + { + // if the setter is not public 'GetSetMethod' will not find it + // but calling 'SetValue' will work, so for backwards compatibility + // we will use it instead + self.setterCache = BuildSetter(self.info); + if (self.setterCache == null) + { + self.setterCacheFailed = true; + } + } + + if (self.setterCacheFailed) + { + self.info.SetValue(co.inst, newval, null); + } + else + { + self.setterCache(co.inst, newval); + } } else { @@ -160,5 +198,54 @@ public static IntPtr tp_repr(IntPtr ob) var self = (PropertyObject)GetManagedObject(ob); return Runtime.PyString_FromString($""); } + + private static Func BuildGetter(PropertyInfo propertyInfo) + { + var methodInfo = propertyInfo.GetGetMethod(); + if (methodInfo == null) + { + // if the getter is not public 'GetGetMethod' will not find it + return null; + } + var obj = Expression.Parameter(typeof(object), "o"); + // Require because we will know at runtime the declaring type + // so 'obj' is declared as typeof(object) + var instance = Expression.Convert(obj, methodInfo.DeclaringType); + + var expressionCall = Expression.Call(instance, methodInfo); + + return Expression.Lambda>( + Expression.Convert(expressionCall, typeof(object)), + obj).Compile(); + } + + private static Action BuildSetter(PropertyInfo propertyInfo) + { + var methodInfo = propertyInfo.GetSetMethod(); + if (methodInfo == null) + { + // if the setter is not public 'GetSetMethod' will not find it + return null; + } + var obj = Expression.Parameter(typeof(object), "o"); + // Require because we will know at runtime the declaring type + // so 'obj' is declared as typeof(object) + var instance = Expression.Convert(obj, methodInfo.DeclaringType); + + var parameters = methodInfo.GetParameters(); + if (parameters.Length != 1) + { + return null; + } + var value = Expression.Parameter(typeof(object)); + var argument = Expression.Convert(value, parameters[0].ParameterType); + + var expressionCall = Expression.Call(instance, methodInfo, argument); + + return Expression.Lambda>( + expressionCall, + obj, + value).Compile(); + } } } diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index 3b8c71efa..42ba1ff39 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -1,11 +1,17 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Dynamic; using System.Linq.Expressions; namespace Python.Runtime { + public interface IPyDisposable : IDisposable + { + IntPtr[] GetTrackedHandles(); + } + /// /// Represents a generic Python object. The methods of this class are /// generally equivalent to the Python "abstract object API". See @@ -13,10 +19,18 @@ namespace Python.Runtime /// PY3: https://docs.python.org/3/c-api/object.html /// for details. /// - public class PyObject : DynamicObject, IEnumerable, IDisposable + public class PyObject : DynamicObject, IEnumerable, IPyDisposable { +#if TRACE_ALLOC + /// + /// Trace stack for PyObject's construction + /// + public StackTrace Traceback { get; private set; } +#endif + protected internal IntPtr obj = IntPtr.Zero; private bool disposed = false; + private bool _finalized = false; /// /// PyObject Constructor @@ -30,6 +44,9 @@ public class PyObject : DynamicObject, IEnumerable, IDisposable public PyObject(IntPtr ptr) { obj = ptr; +#if TRACE_ALLOC + Traceback = new StackTrace(1); +#endif } // Protected default constructor to allow subclasses to manage @@ -37,18 +54,26 @@ public PyObject(IntPtr ptr) protected PyObject() { +#if TRACE_ALLOC + Traceback = new StackTrace(1); +#endif } // Ensure that encapsulated Python object is decref'ed appropriately // when the managed wrapper is garbage-collected. - ~PyObject() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; - - Dispose(); + if (obj == IntPtr.Zero) + { + return; + } + if (_finalized || disposed) + { + return; + } + // Prevent a infinity loop by calling GC.WaitForPendingFinalizers + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); } @@ -101,7 +126,7 @@ public object AsManagedObject(Type t) } return result; } - + /// /// As Method /// @@ -156,6 +181,28 @@ public void Dispose() GC.SuppressFinalize(this); } + /// + /// Unsafe Dispose Method. + /// To be used when already owning the GIL lock. + /// + public void UnsafeDispose() + { + if (!disposed) + { + if (!Runtime.IsFinalizing) + { + Runtime.XDecref(obj); + obj = IntPtr.Zero; + } + disposed = true; + } + GC.SuppressFinalize(this); + } + + public IntPtr[] GetTrackedHandles() + { + return new IntPtr[] { obj }; + } /// /// GetPythonType Method @@ -519,9 +566,9 @@ public virtual void DelItem(int index) /// Returns the length for objects that support the Python sequence /// protocol, or 0 if the object does not support the protocol. /// - public virtual long Length() + public virtual int Length() { - var s = Runtime.PyObject_Size(obj); + int s = Runtime.PyObject_Size(obj); if (s < 0) { Runtime.PyErr_Clear(); diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 8e6957855..f5449f30c 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -22,7 +22,7 @@ public class PyGILAttribute : Attribute } [PyGIL] - public class PyScope : DynamicObject, IDisposable + public class PyScope : DynamicObject, IPyDisposable { public readonly string Name; @@ -37,6 +37,7 @@ public class PyScope : DynamicObject, IDisposable internal readonly IntPtr variables; private bool _isDisposed; + private bool _finalized = false; /// /// The Manager this scope associated with. @@ -525,27 +526,28 @@ public void Dispose() this.OnDispose?.Invoke(this); } - ~PyScope() + public IntPtr[] GetTrackedHandles() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; + return new IntPtr[] { obj }; + } - Dispose(); + ~PyScope() + { + if (_finalized || _isDisposed) + { + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); } } public class PyScopeManager { - public static PyScopeManager Global; + public readonly static PyScopeManager Global = new PyScopeManager(); private Dictionary NamedScopes = new Dictionary(); - internal static void Reset() - { - Global = new PyScopeManager(); - } - internal PyScope NewScope(string name) { if (name == null) diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index c1b663d22..5a62c02bb 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -140,9 +140,9 @@ public static void Initialize() Initialize(setSysArgv: true); } - public static void Initialize(bool setSysArgv = true, bool initSigs = false) + public static void Initialize(bool setSysArgv = true) { - Initialize(Enumerable.Empty(), setSysArgv: setSysArgv, initSigs: initSigs); + Initialize(Enumerable.Empty(), setSysArgv: setSysArgv); } /// @@ -153,9 +153,8 @@ public static void Initialize(bool setSysArgv = true, bool initSigs = false) /// more than once, though initialization will only happen on the /// first call. It is *not* necessary to hold the Python global /// interpreter lock (GIL) to call this method. - /// initSigs can be set to 1 to do default python signal configuration. This will override the way signals are handled by the application. /// - public static void Initialize(IEnumerable args, bool setSysArgv = true, bool initSigs = false) + public static void Initialize(IEnumerable args, bool setSysArgv = true) { if (!initialized) { @@ -165,20 +164,11 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, // during an initial "import clr", and the world ends shortly thereafter. // This is probably masking some bad mojo happening somewhere in Runtime.Initialize(). delegateManager = new DelegateManager(); - Runtime.Initialize(initSigs); + Console.WriteLine("PythonEngine.Initialize(): Runtime.Initialize()..."); + Runtime.Initialize(); initialized = true; Exceptions.Clear(); - // Make sure we clean up properly on app domain unload. - AppDomain.CurrentDomain.DomainUnload += OnDomainUnload; - - // Remember to shut down the runtime. - AddShutdownHandler(Runtime.Shutdown); - - // The global scope gets used implicitly quite early on, remember - // to clear it out when we shut down. - AddShutdownHandler(PyScopeManager.Global.Clear); - if (setSysArgv) { Py.SetArgv(args); @@ -190,9 +180,11 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, string code = "import atexit, clr\n" + "atexit.register(clr._AtExit)\n"; + Console.WriteLine("PythonEngine.Initialize(): register atexit callback..."); PythonEngine.Exec(code); // Load the clr.py resource into the clr module + Console.WriteLine("PythonEngine.Initialize(): GetCLRModule()..."); IntPtr clr = Python.Runtime.ImportHook.GetCLRModule(); IntPtr clr_dict = Runtime.PyModule_GetDict(clr); @@ -204,6 +196,7 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, IntPtr builtins = Runtime.PyEval_GetBuiltins(); Runtime.PyDict_SetItemString(module_globals, "__builtins__", builtins); + Console.WriteLine("PythonEngine.Initialize(): clr GetManifestResourceStream..."); Assembly assembly = Assembly.GetExecutingAssembly(); using (Stream stream = assembly.GetManifestResourceStream("clr.py")) using (var reader = new StreamReader(stream)) @@ -234,11 +227,6 @@ public static void Initialize(IEnumerable args, bool setSysArgv = true, } } - static void OnDomainUnload(object _, EventArgs __) - { - Shutdown(); - } - /// /// A helper to perform initialization from the context of an active /// CPython interpreter process - this bootstraps the managed runtime @@ -311,80 +299,18 @@ public static void Shutdown() if (initialized) { PyScopeManager.Global.Clear(); - - // If the shutdown handlers trigger a domain unload, - // don't call shutdown again. - AppDomain.CurrentDomain.DomainUnload -= OnDomainUnload; - - ExecuteShutdownHandlers(); + Marshal.FreeHGlobal(_pythonHome); + _pythonHome = IntPtr.Zero; + Marshal.FreeHGlobal(_programName); + _programName = IntPtr.Zero; + Marshal.FreeHGlobal(_pythonPath); + _pythonPath = IntPtr.Zero; + Runtime.Shutdown(); initialized = false; } } - /// - /// Called when the engine is shut down. - /// - /// Shutdown handlers are run in reverse order they were added, so that - /// resources available when running a shutdown handler are the same as - /// what was available when it was added. - /// - public delegate void ShutdownHandler(); - - static List ShutdownHandlers = new List(); - - /// - /// Add a function to be called when the engine is shut down. - /// - /// Shutdown handlers are executed in the opposite order they were - /// added, so that you can be sure that everything that was initialized - /// when you added the handler is still initialized when you need to shut - /// down. - /// - /// If the same shutdown handler is added several times, it will be run - /// several times. - /// - /// Don't add shutdown handlers while running a shutdown handler. - /// - public static void AddShutdownHandler(ShutdownHandler handler) - { - ShutdownHandlers.Add(handler); - } - - /// - /// Remove a shutdown handler. - /// - /// If the same shutdown handler is added several times, only the last - /// one is removed. - /// - /// Don't remove shutdown handlers while running a shutdown handler. - /// - public static void RemoveShutdownHandler(ShutdownHandler handler) - { - for (int index = ShutdownHandlers.Count - 1; index >= 0; --index) - { - if (ShutdownHandlers[index] == handler) - { - ShutdownHandlers.RemoveAt(index); - break; - } - } - } - - /// - /// Run all the shutdown handlers. - /// - /// They're run in opposite order they were added. - /// - static void ExecuteShutdownHandlers() - { - while(ShutdownHandlers.Count > 0) - { - var handler = ShutdownHandlers[ShutdownHandlers.Count - 1]; - ShutdownHandlers.RemoveAt(ShutdownHandlers.Count - 1); - handler(); - } - } /// /// AcquireLock Method diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index ded7fbeb5..295a63b3d 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -6,7 +6,7 @@ namespace Python.Runtime /// Provides a managed interface to exceptions thrown by the Python /// runtime. /// - public class PythonException : System.Exception + public class PythonException : System.Exception, IPyDisposable { private IntPtr _pyType = IntPtr.Zero; private IntPtr _pyValue = IntPtr.Zero; @@ -15,14 +15,12 @@ public class PythonException : System.Exception private string _message = ""; private string _pythonTypeName = ""; private bool disposed = false; + private bool _finalized = false; public PythonException() { IntPtr gs = PythonEngine.AcquireLock(); Runtime.PyErr_Fetch(ref _pyType, ref _pyValue, ref _pyTB); - Runtime.XIncref(_pyType); - Runtime.XIncref(_pyValue); - Runtime.XIncref(_pyTB); if (_pyType != IntPtr.Zero && _pyValue != IntPtr.Zero) { string type; @@ -45,11 +43,13 @@ public PythonException() } if (_pyTB != IntPtr.Zero) { - PyObject tb_module = PythonEngine.ImportModule("traceback"); - Runtime.XIncref(_pyTB); - using (var pyTB = new PyObject(_pyTB)) + using (PyObject tb_module = PythonEngine.ImportModule("traceback")) { - _tb = tb_module.InvokeMethod("format_tb", pyTB).ToString(); + Runtime.XIncref(_pyTB); + using (var pyTB = new PyObject(_pyTB)) + { + _tb = tb_module.InvokeMethod("format_tb", pyTB).ToString(); + } } } PythonEngine.ReleaseLock(gs); @@ -60,11 +60,12 @@ public PythonException() ~PythonException() { - // We needs to disable Finalizers until it's valid implementation. - // Current implementation can produce low probability floating bugs. - return; - - Dispose(); + if (_finalized || disposed) + { + return; + } + _finalized = true; + Finalizer.Instance.AddFinalizedObject(this); } /// @@ -132,7 +133,7 @@ public override string Message /// public override string StackTrace { - get { return _tb; } + get { return _tb + base.StackTrace; } } /// @@ -173,6 +174,11 @@ public void Dispose() } } + public IntPtr[] GetTrackedHandles() + { + return new IntPtr[] { _pyType, _pyValue, _pyTB }; + } + /// /// Matches Method /// diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index e708a54ac..3e35b13ab 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "2.4.0.dev0" +__version__ = "1.0.5.30" class clrproperty(object): diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 863eb4034..06740273a 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -2,7 +2,7 @@ using System.Runtime.InteropServices; using System.Security; using System.Text; -using System.Collections.Generic; +using System.Threading; namespace Python.Runtime { @@ -22,7 +22,7 @@ public static IntPtr LoadLibrary(string fileName) } #elif MONO_OSX private static int RTLD_GLOBAL = 0x8; - private const string NativeDll = "/usr/lib/libSystem.dylib"; + private const string NativeDll = "/usr/lib/libSystem.dylib" private static IntPtr RTLD_DEFAULT = new IntPtr(-2); public static IntPtr LoadLibrary(string fileName) @@ -105,7 +105,7 @@ public static IntPtr GetProcAddress(IntPtr dllHandle, string name) public class Runtime { // C# compiler copies constants to the assemblies that references this library. - // We needs to replace all public constants to static readonly fields to allow + // We needs to replace all public constants to static readonly fields to allow // binary substitution of different Python.Runtime.dll builds in a target application. public static int UCS => _UCS; @@ -131,7 +131,7 @@ public class Runtime #endif // C# compiler copies constants to the assemblies that references this library. - // We needs to replace all public constants to static readonly fields to allow + // We needs to replace all public constants to static readonly fields to allow // binary substitution of different Python.Runtime.dll builds in a target application. public static string pyversion => _pyversion; @@ -149,7 +149,7 @@ public class Runtime #elif PYTHON36 internal const string _pyversion = "3.6"; internal const string _pyver = "36"; -#elif PYTHON37 +#elif PYTHON37 // TODO: Add `interop37.cs` after PY37 is released internal const string _pyversion = "3.7"; internal const string _pyver = "37"; #else @@ -174,7 +174,7 @@ public class Runtime #endif // C# compiler copies constants to the assemblies that references this library. - // We needs to replace all public constants to static readonly fields to allow + // We needs to replace all public constants to static readonly fields to allow // binary substitution of different Python.Runtime.dll builds in a target application. public static readonly string PythonDLL = _PythonDll; @@ -196,71 +196,11 @@ public class Runtime // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows) internal static bool IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; - /// - /// Operating system type as reported by Python. - /// - public enum OperatingSystemType - { - Windows, - Darwin, - Linux, - Other - } - - static readonly Dictionary OperatingSystemTypeMapping = new Dictionary() - { - { "Windows", OperatingSystemType.Windows }, - { "Darwin", OperatingSystemType.Darwin }, - { "Linux", OperatingSystemType.Linux }, - }; - - /// - /// Gets the operating system as reported by python's platform.system(). - /// - public static OperatingSystemType OperatingSystem { get; private set; } - - /// - /// Gets the operating system as reported by python's platform.system(). - /// - public static string OperatingSystemName { get; private set; } - - public enum MachineType - { - i386, - x86_64, - Other - }; - - /// - /// Map lower-case version of the python machine name to the processor - /// type. There are aliases, e.g. x86_64 and amd64 are two names for - /// the same thing. Make sure to lower-case the search string, because - /// capitalization can differ. - /// - static readonly Dictionary MachineTypeMapping = new Dictionary() - { - ["i386"] = MachineType.i386, - ["i686"] = MachineType.i386, - ["x86"] = MachineType.i386, - ["x86_64"] = MachineType.x86_64, - ["amd64"] = MachineType.x86_64, - ["x64"] = MachineType.x86_64, - ["em64t"] = MachineType.x86_64, - }; - - /// - /// Gets the machine architecture as reported by python's platform.machine(). - /// - public static MachineType Machine { get; private set; }/* set in Initialize using python's platform.machine */ - - /// - /// Gets the machine architecture as reported by python's platform.machine(). - /// - public static string MachineName { get; private set; } - internal static bool IsPython2 = pyversionnumber < 30; internal static bool IsPython3 = pyversionnumber >= 30; + public static int MainManagedThreadId { get; private set; } + /// /// Encoding to use to convert Unicode to/from Managed to Native /// @@ -269,27 +209,22 @@ public enum MachineType /// /// Initialize the runtime... /// - internal static void Initialize(bool initSigs = false) + internal static void Initialize() { if (Py_IsInitialized() == 0) { - Py_InitializeEx(initSigs ? 1 : 0); + Console.WriteLine("Runtime.Initialize(): Py_Initialize..."); + Py_Initialize(); + MainManagedThreadId = Thread.CurrentThread.ManagedThreadId; } if (PyEval_ThreadsInitialized() == 0) { + Console.WriteLine("Runtime.Initialize(): PyEval_InitThreads..."); PyEval_InitThreads(); } - IsFinalizing = false; - - CLRModule.Reset(); - GenericUtil.Reset(); - PyScopeManager.Reset(); - ClassManager.Reset(); - ClassDerivedObject.Reset(); - TypeManager.Reset(); - + Console.WriteLine("Runtime.Initialize(): Initialize types..."); IntPtr op; IntPtr dict; if (IsPython3) @@ -368,6 +303,14 @@ internal static void Initialize(bool initSigs = false) PyFloatType = PyObject_Type(op); XDecref(op); + IntPtr decimalMod = PyImport_ImportModule("decimal"); + IntPtr decimalCtor = PyObject_GetAttrString(decimalMod, "Decimal"); + op = PyObject_CallObject(decimalCtor, IntPtr.Zero); + PyDecimalType = PyObject_Type(op); + XDecref(op); + XDecref(decimalMod); + XDecref(decimalCtor); + #if PYTHON3 PyClassType = IntPtr.Zero; PyInstanceType = IntPtr.Zero; @@ -386,6 +329,7 @@ internal static void Initialize(bool initSigs = false) XDecref(c); XDecref(d); #endif + Console.WriteLine("Runtime.Initialize(): Initialize types end."); Error = new IntPtr(-1); @@ -403,11 +347,7 @@ internal static void Initialize(bool initSigs = false) NativeMethods.FreeLibrary(dllLocal); } #endif - // Initialize data about the platform we're running on. We need - // this for the type manager and potentially other details. Must - // happen after caching the python types, above. - InitializePlatformData(); - + Console.WriteLine("Runtime.Initialize(): AssemblyManager.Initialize()..."); // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); PyCLRMetaType = MetaType.Initialize(); @@ -421,61 +361,16 @@ internal static void Initialize(bool initSigs = false) IntPtr item = PyString_FromString(rtdir); PyList_Append(path, item); XDecref(item); + Console.WriteLine("Runtime.Initialize(): AssemblyManager.UpdatePath()..."); AssemblyManager.UpdatePath(); } - /// - /// Initializes the data about platforms. - /// - /// This must be the last step when initializing the runtime: - /// GetManagedString needs to have the cached values for types. - /// But it must run before initializing anything outside the runtime - /// because those rely on the platform data. - /// - private static void InitializePlatformData() - { - IntPtr op; - IntPtr fn; - IntPtr platformModule = PyImport_ImportModule("platform"); - IntPtr emptyTuple = PyTuple_New(0); - - fn = PyObject_GetAttrString(platformModule, "system"); - op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - OperatingSystemName = GetManagedString(op); - XDecref(op); - XDecref(fn); - - fn = PyObject_GetAttrString(platformModule, "machine"); - op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - MachineName = GetManagedString(op); - XDecref(op); - XDecref(fn); - - XDecref(emptyTuple); - XDecref(platformModule); - - // Now convert the strings into enum values so we can do switch - // statements rather than constant parsing. - OperatingSystemType OSType; - if (!OperatingSystemTypeMapping.TryGetValue(OperatingSystemName, out OSType)) - { - OSType = OperatingSystemType.Other; - } - OperatingSystem = OSType; - - MachineType MType; - if (!MachineTypeMapping.TryGetValue(MachineName.ToLower(), out MType)) - { - MType = MachineType.Other; - } - Machine = MType; - } - internal static void Shutdown() { AssemblyManager.Shutdown(); Exceptions.Shutdown(); ImportHook.Shutdown(); + Finalizer.Shutdown(); Py_Finalize(); } @@ -512,6 +407,7 @@ internal static int AtExit() internal static IntPtr PyBoolType; internal static IntPtr PyNoneType; internal static IntPtr PyTypeType; + internal static IntPtr PyDecimalType; #if PYTHON3 internal static IntPtr PyBytesType; @@ -540,7 +436,7 @@ internal static int AtExit() /// internal static void CheckExceptionOccurred() { - if (PyErr_Occurred() != IntPtr.Zero) + if (PyErr_Occurred() != 0) { throw new PythonException(); } @@ -548,7 +444,7 @@ internal static void CheckExceptionOccurred() internal static IntPtr ExtendTuple(IntPtr t, params IntPtr[] args) { - var size = PyTuple_Size(t); + int size = PyTuple_Size(t); int add = args.Length; IntPtr item; @@ -591,7 +487,7 @@ internal static Type[] PythonArgsToTypeArray(IntPtr arg, bool mangleObjects) free = true; } - var n = PyTuple_Size(args); + int n = PyTuple_Size(args); var types = new Type[n]; Type t = null; @@ -727,9 +623,6 @@ internal static unsafe long Refcount(IntPtr op) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void Py_Initialize(); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void Py_InitializeEx(int initsigs); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int Py_IsInitialized(); @@ -1025,13 +918,8 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyObject_Not(IntPtr pointer); - internal static long PyObject_Size(IntPtr pointer) - { - return (long) _PyObject_Size(pointer); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyObject_Size")] - private static extern IntPtr _PyObject_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyObject_Size(IntPtr pointer); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyObject_Hash(IntPtr op); @@ -1261,61 +1149,26 @@ internal static bool PyFloat_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern bool PySequence_Check(IntPtr pointer); - internal static IntPtr PySequence_GetItem(IntPtr pointer, long index) - { - return PySequence_GetItem(pointer, new IntPtr(index)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_GetItem(IntPtr pointer, IntPtr index); - - internal static int PySequence_SetItem(IntPtr pointer, long index, IntPtr value) - { - return PySequence_SetItem(pointer, new IntPtr(index), value); - } + internal static extern IntPtr PySequence_GetItem(IntPtr pointer, int index); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_SetItem(IntPtr pointer, IntPtr index, IntPtr value); - - internal static int PySequence_DelItem(IntPtr pointer, long index) - { - return PySequence_DelItem(pointer, new IntPtr(index)); - } + internal static extern int PySequence_SetItem(IntPtr pointer, int index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_DelItem(IntPtr pointer, IntPtr index); - - internal static IntPtr PySequence_GetSlice(IntPtr pointer, long i1, long i2) - { - return PySequence_GetSlice(pointer, new IntPtr(i1), new IntPtr(i2)); - } + internal static extern int PySequence_DelItem(IntPtr pointer, int index); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_GetSlice(IntPtr pointer, IntPtr i1, IntPtr i2); - - internal static int PySequence_SetSlice(IntPtr pointer, long i1, long i2, IntPtr v) - { - return PySequence_SetSlice(pointer, new IntPtr(i1), new IntPtr(i2), v); - } + internal static extern IntPtr PySequence_GetSlice(IntPtr pointer, int i1, int i2); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_SetSlice(IntPtr pointer, IntPtr i1, IntPtr i2, IntPtr v); - - internal static int PySequence_DelSlice(IntPtr pointer, long i1, long i2) - { - return PySequence_DelSlice(pointer, new IntPtr(i1), new IntPtr(i2)); - } + internal static extern int PySequence_SetSlice(IntPtr pointer, int i1, int i2, IntPtr v); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PySequence_DelSlice(IntPtr pointer, IntPtr i1, IntPtr i2); - - internal static long PySequence_Size(IntPtr pointer) - { - return (long) _PySequence_Size(pointer); - } + internal static extern int PySequence_DelSlice(IntPtr pointer, int i1, int i2); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Size")] - private static extern IntPtr _PySequence_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PySequence_Size(IntPtr pointer); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PySequence_Contains(IntPtr pointer, IntPtr item); @@ -1323,24 +1176,14 @@ internal static long PySequence_Size(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PySequence_Concat(IntPtr pointer, IntPtr other); - internal static IntPtr PySequence_Repeat(IntPtr pointer, long count) - { - return PySequence_Repeat(pointer, new IntPtr(count)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PySequence_Repeat(IntPtr pointer, IntPtr count); + internal static extern IntPtr PySequence_Repeat(IntPtr pointer, int count); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PySequence_Index(IntPtr pointer, IntPtr item); - internal static long PySequence_Count(IntPtr pointer, IntPtr value) - { - return (long) _PySequence_Count(pointer, value); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Count")] - private static extern IntPtr _PySequence_Count(IntPtr pointer, IntPtr value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PySequence_Count(IntPtr pointer, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PySequence_Tuple(IntPtr pointer); @@ -1366,57 +1209,33 @@ internal static bool PyString_Check(IntPtr ob) internal static IntPtr PyString_FromString(string value) { -#if PYTHON3 - return PyUnicode_FromKindAndData(_UCS, value, value.Length); -#elif PYTHON2 return PyString_FromStringAndSize(value, value.Length); -#endif } #if PYTHON3 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyBytes_FromString(string op); - internal static long PyBytes_Size(IntPtr op) - { - return (long) _PyBytes_Size(op); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyBytes_Size")] - private static extern IntPtr _PyBytes_Size(IntPtr op); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyBytes_Size(IntPtr op); internal static IntPtr PyBytes_AS_STRING(IntPtr ob) { return ob + BytesOffset.ob_sval; } - internal static IntPtr PyString_FromStringAndSize(string value, long size) - { - return _PyString_FromStringAndSize(value, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyUnicode_FromStringAndSize")] - internal static extern IntPtr _PyString_FromStringAndSize( + internal static extern IntPtr PyString_FromStringAndSize( [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string value, - IntPtr size + int size ); - internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size) - { - return PyUnicode_FromStringAndSize(value, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyUnicode_FromStringAndSize(IntPtr value, IntPtr size); + internal static extern IntPtr PyUnicode_FromStringAndSize(IntPtr value, int size); #elif PYTHON2 - internal static IntPtr PyString_FromStringAndSize(string value, long size) - { - return PyString_FromStringAndSize(value, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyString_FromStringAndSize(string value, IntPtr size); + internal static extern IntPtr PyString_FromStringAndSize(string value, int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyString_AsString(IntPtr op); @@ -1437,30 +1256,20 @@ internal static bool PyUnicode_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err); - internal static IntPtr PyUnicode_FromKindAndData(int kind, string s, long size) - { - return PyUnicode_FromKindAndData(kind, s, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyUnicode_FromKindAndData( + internal static extern IntPtr PyUnicode_FromKindAndData( int kind, [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UcsMarshaler))] string s, - IntPtr size + int size ); - internal static IntPtr PyUnicode_FromUnicode(string s, long size) + internal static IntPtr PyUnicode_FromUnicode(string s, int size) { return PyUnicode_FromKindAndData(_UCS, s, size); } - internal static long PyUnicode_GetSize(IntPtr ob) - { - return (long)_PyUnicode_GetSize(ob); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyUnicode_GetSize")] - private static extern IntPtr _PyUnicode_GetSize(IntPtr ob); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyUnicode_GetSize(IntPtr ob); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyUnicode_AsUnicode(IntPtr ob); @@ -1476,26 +1285,16 @@ internal static long PyUnicode_GetSize(IntPtr ob) EntryPoint = PyUnicodeEntryPoint + "FromEncodedObject")] internal static extern IntPtr PyUnicode_FromEncodedObject(IntPtr ob, IntPtr enc, IntPtr err); - internal static IntPtr PyUnicode_FromUnicode(string s, long size) - { - return PyUnicode_FromUnicode(s, new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = PyUnicodeEntryPoint + "FromUnicode")] - private static extern IntPtr PyUnicode_FromUnicode( + internal static extern IntPtr PyUnicode_FromUnicode( [MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(UcsMarshaler))] string s, - IntPtr size + int size ); - internal static long PyUnicode_GetSize(IntPtr ob) - { - return (long) _PyUnicode_GetSize(ob); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = PyUnicodeEntryPoint + "GetSize")] - internal static extern IntPtr _PyUnicode_GetSize(IntPtr ob); + internal static extern int PyUnicode_GetSize(IntPtr ob); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = PyUnicodeEntryPoint + "AsUnicode")] @@ -1538,7 +1337,7 @@ internal static string GetManagedString(IntPtr op) if (type == PyUnicodeType) { IntPtr p = PyUnicode_AsUnicode(op); - int length = (int)PyUnicode_GetSize(op); + int length = PyUnicode_GetSize(op); int size = length * _UCS; var buffer = new byte[size]; @@ -1604,13 +1403,8 @@ internal static bool PyDict_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyDict_Clear(IntPtr pointer); - internal static long PyDict_Size(IntPtr pointer) - { - return (long) _PyDict_Size(pointer); - } - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyDict_Size")] - internal static extern IntPtr _PyDict_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyDict_Size(IntPtr pointer); //==================================================================== @@ -1622,40 +1416,20 @@ internal static bool PyList_Check(IntPtr ob) return PyObject_TYPE(ob) == PyListType; } - internal static IntPtr PyList_New(long size) - { - return PyList_New(new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_New(IntPtr size); + internal static extern IntPtr PyList_New(int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyList_AsTuple(IntPtr pointer); - internal static IntPtr PyList_GetItem(IntPtr pointer, long index) - { - return PyList_GetItem(pointer, new IntPtr(index)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_GetItem(IntPtr pointer, IntPtr index); - - internal static int PyList_SetItem(IntPtr pointer, long index, IntPtr value) - { - return PyList_SetItem(pointer, new IntPtr(index), value); - } + internal static extern IntPtr PyList_GetItem(IntPtr pointer, int index); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_SetItem(IntPtr pointer, IntPtr index, IntPtr value); - - internal static int PyList_Insert(IntPtr pointer, long index, IntPtr value) - { - return PyList_Insert(pointer, new IntPtr(index), value); - } + internal static extern int PyList_SetItem(IntPtr pointer, int index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_Insert(IntPtr pointer, IntPtr index, IntPtr value); + internal static extern int PyList_Insert(IntPtr pointer, int index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyList_Append(IntPtr pointer, IntPtr value); @@ -1666,29 +1440,15 @@ internal static int PyList_Insert(IntPtr pointer, long index, IntPtr value) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyList_Sort(IntPtr pointer); - internal static IntPtr PyList_GetSlice(IntPtr pointer, long start, long end) - { - return PyList_GetSlice(pointer, new IntPtr(start), new IntPtr(end)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_GetSlice(IntPtr pointer, IntPtr start, IntPtr end); - - internal static int PyList_SetSlice(IntPtr pointer, long start, long end, IntPtr value) - { - return PyList_SetSlice(pointer, new IntPtr(start), new IntPtr(end), value); - } + internal static extern IntPtr PyList_GetSlice(IntPtr pointer, int start, int end); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_SetSlice(IntPtr pointer, IntPtr start, IntPtr end, IntPtr value); + internal static extern int PyList_SetSlice(IntPtr pointer, int start, int end, IntPtr value); - internal static long PyList_Size(IntPtr pointer) - { - return (long) _PyList_Size(pointer); - } + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyList_Size(IntPtr pointer); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyList_Size")] - private static extern IntPtr _PyList_Size(IntPtr pointer); //==================================================================== // Python tuple API @@ -1699,45 +1459,20 @@ internal static bool PyTuple_Check(IntPtr ob) return PyObject_TYPE(ob) == PyTupleType; } - internal static IntPtr PyTuple_New(long size) - { - return PyTuple_New(new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_New(IntPtr size); - - internal static IntPtr PyTuple_GetItem(IntPtr pointer, long index) - { - return PyTuple_GetItem(pointer, new IntPtr(index)); - } + internal static extern IntPtr PyTuple_New(int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_GetItem(IntPtr pointer, IntPtr index); - - internal static int PyTuple_SetItem(IntPtr pointer, long index, IntPtr value) - { - return PyTuple_SetItem(pointer, new IntPtr(index), value); - } + internal static extern IntPtr PyTuple_GetItem(IntPtr pointer, int index); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyTuple_SetItem(IntPtr pointer, IntPtr index, IntPtr value); - - internal static IntPtr PyTuple_GetSlice(IntPtr pointer, long start, long end) - { - return PyTuple_GetSlice(pointer, new IntPtr(start), new IntPtr(end)); - } + internal static extern int PyTuple_SetItem(IntPtr pointer, int index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyTuple_GetSlice(IntPtr pointer, IntPtr start, IntPtr end); - - internal static long PyTuple_Size(IntPtr pointer) - { - return (long) _PyTuple_Size(pointer); - } + internal static extern IntPtr PyTuple_GetSlice(IntPtr pointer, int start, int end); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyTuple_Size")] - private static extern IntPtr _PyTuple_Size(IntPtr pointer); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyTuple_Size(IntPtr pointer); //==================================================================== @@ -1840,16 +1575,16 @@ internal static bool PyObject_TypeCheck(IntPtr ob, IntPtr tp) return (t == tp) || PyType_IsSubtype(t, tp); } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kw); - - internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) + internal static bool PyObjectType_TypeCheck(IntPtr type, IntPtr tp) { - return PyType_GenericAlloc(type, new IntPtr(n)); + return (type == tp) || PyType_IsSubtype(type, tp); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyType_GenericAlloc(IntPtr type, IntPtr n); + internal static extern IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kw); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyType_GenericAlloc(IntPtr type, int n); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyType_Ready(IntPtr type); @@ -1883,21 +1618,11 @@ internal static IntPtr PyType_GenericAlloc(IntPtr type, long n) // Python memory API //==================================================================== - internal static IntPtr PyMem_Malloc(long size) - { - return PyMem_Malloc(new IntPtr(size)); - } - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyMem_Malloc(IntPtr size); - - internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) - { - return PyMem_Realloc(ptr, new IntPtr(size)); - } + internal static extern IntPtr PyMem_Malloc(int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyMem_Realloc(IntPtr ptr, IntPtr size); + internal static extern IntPtr PyMem_Realloc(IntPtr ptr, int size); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyMem_Free(IntPtr ptr); @@ -1929,7 +1654,7 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) internal static extern void PyErr_NormalizeException(IntPtr ob, IntPtr val, IntPtr tb); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyErr_Occurred(); + internal static extern int PyErr_Occurred(); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_Fetch(ref IntPtr ob, ref IntPtr val, ref IntPtr tb); @@ -1953,5 +1678,11 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyMethod_Function(IntPtr ob); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int Py_AddPendingCall(IntPtr func, IntPtr arg); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int Py_MakePendingCalls(); } } diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index d19c8737f..9b8f95649 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -21,10 +21,6 @@ static TypeManager() cache = new Dictionary(128); } - public static void Reset() - { - cache = new Dictionary(128); - } /// /// Given a managed Type derived from ExtensionType, get the handle to @@ -83,7 +79,7 @@ internal static IntPtr CreateType(Type impl) // Set tp_basicsize to the size of our managed instance objects. Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); - var offset = (IntPtr)ObjectOffset.DictOffset(type); + var offset = (IntPtr)ObjectOffset.TypeDictOffset(type); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); InitializeSlots(type, impl); @@ -121,7 +117,6 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) IntPtr base_ = IntPtr.Zero; int ob_size = ObjectOffset.Size(Runtime.PyTypeType); - int tp_dictoffset = ObjectOffset.DictOffset(Runtime.PyTypeType); // XXX Hack, use a different base class for System.Exception // Python 2.5+ allows new style class exceptions but they *must* @@ -129,9 +124,10 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) if (typeof(Exception).IsAssignableFrom(clrType)) { ob_size = ObjectOffset.Size(Exceptions.Exception); - tp_dictoffset = ObjectOffset.DictOffset(Exceptions.Exception); } + int tp_dictoffset = ob_size + ManagedDataOffsets.ob_dict; + if (clrType == typeof(Exception)) { base_ = Exceptions.Exception; @@ -451,225 +447,6 @@ internal static IntPtr AllocateTypeObject(string name) } - #region Native Code Page - /// - /// Initialized by InitializeNativeCodePage. - /// - /// This points to a page of memory allocated using mmap or VirtualAlloc - /// (depending on the system), and marked read and execute (not write). - /// Very much on purpose, the page is *not* released on a shutdown and - /// is instead leaked. See the TestDomainReload test case. - /// - /// The contents of the page are two native functions: one that returns 0, - /// one that returns 1. - /// - /// If python didn't keep its gc list through a Py_Finalize we could remove - /// this entire section. - /// - internal static IntPtr NativeCodePage = IntPtr.Zero; - - /// - /// Structure to describe native code. - /// - /// Use NativeCode.Active to get the native code for the current platform. - /// - /// Generate the code by creating the following C code: - /// - /// int Return0() { return 0; } - /// int Return1() { return 1; } - /// - /// Then compiling on the target platform, e.g. with gcc or clang: - /// cc -c -fomit-frame-pointer -O2 foo.c - /// And then analyzing the resulting functions with a hex editor, e.g.: - /// objdump -disassemble foo.o - /// - internal class NativeCode - { - /// - /// The code, as a string of bytes. - /// - public byte[] Code { get; private set; } - - /// - /// Where does the "return 0" function start? - /// - public int Return0 { get; private set; } - - /// - /// Where does the "return 1" function start? - /// - public int Return1 { get; private set; } - - public static NativeCode Active - { - get - { - switch(Runtime.Machine) - { - case Runtime.MachineType.i386: - return I386; - case Runtime.MachineType.x86_64: - return X86_64; - default: - throw new NotImplementedException($"No support for {Runtime.MachineName}"); - } - } - } - - /// - /// Code for x86_64. See the class comment for how it was generated. - /// - public static readonly NativeCode X86_64 = new NativeCode() - { - Return0 = 0x10, - Return1 = 0, - Code = new byte[] - { - // First Return1: - 0xb8, 0x01, 0x00, 0x00, 0x00, // movl $1, %eax - 0xc3, // ret - - // Now some padding so that Return0 can be 16-byte-aligned. - // I put Return1 first so there's not as much padding to type in. - 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, // nop - - // Now Return0. - 0x31, 0xc0, // xorl %eax, %eax - 0xc3, // ret - } - }; - - /// - /// Code for X86. - /// - /// It's bitwise identical to X86_64, so we just point to it. - /// - /// - public static readonly NativeCode I386 = X86_64; - } - - /// - /// Platform-dependent mmap and mprotect. - /// - internal interface IMemoryMapper - { - /// - /// Map at least numBytes of memory. Mark the page read-write (but not exec). - /// - IntPtr MapWriteable(int numBytes); - - /// - /// Sets the mapped memory to be read-exec (but not write). - /// - void SetReadExec(IntPtr mappedMemory, int numBytes); - } - - class WindowsMemoryMapper : IMemoryMapper - { - const UInt32 MEM_COMMIT = 0x1000; - const UInt32 MEM_RESERVE = 0x2000; - const UInt32 PAGE_READWRITE = 0x04; - const UInt32 PAGE_EXECUTE_READ = 0x20; - - [DllImport("kernel32.dll")] - static extern IntPtr VirtualAlloc(IntPtr lpAddress, IntPtr dwSize, UInt32 flAllocationType, UInt32 flProtect); - - [DllImport("kernel32.dll")] - static extern bool VirtualProtect(IntPtr lpAddress, IntPtr dwSize, UInt32 flNewProtect, out UInt32 lpflOldProtect); - - public IntPtr MapWriteable(int numBytes) - { - return VirtualAlloc(IntPtr.Zero, new IntPtr(numBytes), - MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - UInt32 _; - VirtualProtect(mappedMemory, new IntPtr(numBytes), PAGE_EXECUTE_READ, out _); - } - } - - class UnixMemoryMapper : IMemoryMapper - { - const int PROT_READ = 0x1; - const int PROT_WRITE = 0x2; - const int PROT_EXEC = 0x4; - - const int MAP_PRIVATE = 0x2; - int MAP_ANONYMOUS - { - get - { - switch (Runtime.OperatingSystem) - { - case Runtime.OperatingSystemType.Darwin: - return 0x1000; - case Runtime.OperatingSystemType.Linux: - return 0x20; - default: - throw new NotImplementedException($"mmap is not supported on {Runtime.OperatingSystemName}"); - } - } - } - - [DllImport("libc")] - static extern IntPtr mmap(IntPtr addr, IntPtr len, int prot, int flags, int fd, IntPtr offset); - - [DllImport("libc")] - static extern int mprotect(IntPtr addr, IntPtr len, int prot); - - public IntPtr MapWriteable(int numBytes) - { - // MAP_PRIVATE must be set on linux, even though MAP_ANON implies it. - // It doesn't hurt on darwin, so just do it. - return mmap(IntPtr.Zero, new IntPtr(numBytes), PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, IntPtr.Zero); - } - - public void SetReadExec(IntPtr mappedMemory, int numBytes) - { - mprotect(mappedMemory, new IntPtr(numBytes), PROT_READ | PROT_EXEC); - } - } - - internal static IMemoryMapper CreateMemoryMapper() - { - switch (Runtime.OperatingSystem) - { - case Runtime.OperatingSystemType.Darwin: - case Runtime.OperatingSystemType.Linux: - return new UnixMemoryMapper(); - case Runtime.OperatingSystemType.Windows: - return new WindowsMemoryMapper(); - default: - throw new NotImplementedException($"No support for {Runtime.OperatingSystemName}"); - } - } - - /// - /// Initializes the native code page. - /// - /// Safe to call if we already initialized (this function is idempotent). - /// - /// - internal static void InitializeNativeCodePage() - { - // Do nothing if we already initialized. - if (NativeCodePage != IntPtr.Zero) - { - return; - } - - // Allocate the page, write the native code into it, then set it - // to be executable. - IMemoryMapper mapper = CreateMemoryMapper(); - int codeLength = NativeCode.Active.Code.Length; - NativeCodePage = mapper.MapWriteable(codeLength); - Marshal.Copy(NativeCode.Active.Code, 0, NativeCodePage, codeLength); - mapper.SetReadExec(NativeCodePage, codeLength); - } -#endregion - /// /// Given a newly allocated Python type object and a managed Type that /// provides the implementation for the type, connect the type slots of @@ -677,10 +454,8 @@ internal static void InitializeNativeCodePage() /// internal static void InitializeSlots(IntPtr type, Type impl) { - // We work from the most-derived class up; make sure to get - // the most-derived slot and not to override it with a base - // class's slot. - var seen = new HashSet(); + var seen = new Hashtable(8); + Type offsetType = typeof(TypeOffset); while (impl != null) { @@ -698,52 +473,24 @@ internal static void InitializeSlots(IntPtr type, Type impl) continue; } - if (seen.Contains(name)) + if (seen[name] != null) { continue; } - InitializeSlot(type, Interop.GetThunk(method), name); + FieldInfo fi = offsetType.GetField(name); + var offset = (int)fi.GetValue(offsetType); + + IntPtr slot = Interop.GetThunk(method); + Marshal.WriteIntPtr(type, offset, slot); - seen.Add(name); + seen[name] = 1; } impl = impl.BaseType; } - - // See the TestDomainReload test: there was a crash related to - // the gc-related slots. They always return 0 or 1 because we don't - // really support gc: - // tp_traverse (returns 0) - // tp_clear (returns 0) - // tp_is_gc (returns 1) - // We can't do without: python really wants those slots to exist. - // We can't implement those in C# because the application domain - // can be shut down and the memory released. - InitializeNativeCodePage(); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return0, "tp_traverse"); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return0, "tp_clear"); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return1, "tp_is_gc"); } - /// - /// Helper for InitializeSlots. - /// - /// Initializes one slot to point to a function pointer. - /// The function pointer might be a thunk for C#, or it may be - /// an address in the NativeCodePage. - /// - /// Type being initialized. - /// Function pointer. - /// Name of the method. - static void InitializeSlot(IntPtr type, IntPtr slot, string name) - { - Type typeOffset = typeof(TypeOffset); - FieldInfo fi = typeOffset.GetField(name); - var offset = (int)fi.GetValue(typeOffset); - - Marshal.WriteIntPtr(type, offset, slot); - } /// /// Given a newly allocated Python type object and a managed Type that diff --git a/src/testing/InheritanceTest.cs b/src/testing/InheritanceTest.cs deleted file mode 100644 index 529703b3c..000000000 --- a/src/testing/InheritanceTest.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; - -namespace Python.Test -{ - public class BaseClass - { - public bool IsBase() => true; - } - - public class DerivedClass : BaseClass - { - public new bool IsBase() => false; - } -} diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index 27639ed5a..434da99e1 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -1,5 +1,5 @@ - - + + Debug AnyCPU @@ -9,7 +9,7 @@ Python.Test bin\Python.Test.xml bin\ - v4.0 + v4.5.2 1591,0067 ..\..\ @@ -70,6 +70,7 @@ pdbonly + @@ -83,7 +84,6 @@ - diff --git a/src/testing/conversiontest.cs b/src/testing/conversiontest.cs index 06ab7cb4e..36294594c 100644 --- a/src/testing/conversiontest.cs +++ b/src/testing/conversiontest.cs @@ -60,19 +60,4 @@ public string GetValue() return value; } } - - public class UnicodeString - { - public string value = "안녕"; - - public string GetString() - { - return value; - } - - public override string ToString() - { - return value; - } - } } diff --git a/src/testing/dictionarytest.cs b/src/testing/dictionarytest.cs new file mode 100644 index 000000000..a7fa3497d --- /dev/null +++ b/src/testing/dictionarytest.cs @@ -0,0 +1,106 @@ +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +namespace Python.Test +{ + /// + /// Supports units tests for dictionary __contains__ and __len__ + /// + public class PublicDictionaryTest + { + public IDictionary items; + + public PublicDictionaryTest() + { + items = new int[5] { 0, 1, 2, 3, 4 } + .ToDictionary(k => k.ToString(), v => v); + } + } + + + public class ProtectedDictionaryTest + { + protected IDictionary items; + + public ProtectedDictionaryTest() + { + items = new int[5] { 0, 1, 2, 3, 4 } + .ToDictionary(k => k.ToString(), v => v); + } + } + + + public class InternalDictionaryTest + { + internal IDictionary items; + + public InternalDictionaryTest() + { + items = new int[5] { 0, 1, 2, 3, 4 } + .ToDictionary(k => k.ToString(), v => v); + } + } + + + public class PrivateDictionaryTest + { + private IDictionary items; + + public PrivateDictionaryTest() + { + items = new int[5] { 0, 1, 2, 3, 4 } + .ToDictionary(k => k.ToString(), v => v); + } + } + + public class InheritedDictionaryTest : IDictionary + { + private readonly IDictionary items; + + public InheritedDictionaryTest() + { + items = new int[5] { 0, 1, 2, 3, 4 } + .ToDictionary(k => k.ToString(), v => v); + } + + public int this[string key] + { + get { return items[key]; } + set { items[key] = value; } + } + + public ICollection Keys => items.Keys; + + public ICollection Values => items.Values; + + public int Count => items.Count; + + public bool IsReadOnly => false; + + public void Add(string key, int value) => items.Add(key, value); + + public void Add(KeyValuePair item) => items.Add(item); + + public void Clear() => items.Clear(); + + public bool Contains(KeyValuePair item) => items.Contains(item); + + public bool ContainsKey(string key) => items.ContainsKey(key); + + public void CopyTo(KeyValuePair[] array, int arrayIndex) + { + items.CopyTo(array, arrayIndex); + } + + public IEnumerator> GetEnumerator() => items.GetEnumerator(); + + public bool Remove(string key) => items.Remove(key); + + public bool Remove(KeyValuePair item) => items.Remove(item); + + public bool TryGetValue(string key, out int value) => items.TryGetValue(key, out value); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } +} diff --git a/src/tests/importtest.py b/src/tests/importtest.py deleted file mode 100644 index fe93764e9..000000000 --- a/src/tests/importtest.py +++ /dev/null @@ -1,13 +0,0 @@ -# -*- coding: utf-8 -*- - -import sys -try: - del sys.modules["System.IO"] -except KeyError: - pass - -assert "FileStream" not in globals() -import System.IO -from System.IO import * - -assert "FileStream" in globals() diff --git a/src/tests/test_class.py b/src/tests/test_class.py index 612ce442e..68773508b 100644 --- a/src/tests/test_class.py +++ b/src/tests/test_class.py @@ -281,14 +281,3 @@ def PyCallback(self, self2): testobj.DoCallback() assert testobj.PyCallbackWasCalled assert testobj.SameReference - - -def test_method_inheritance(): - """Ensure that we call the overridden method instead of the one provided in - the base class.""" - - base = Test.BaseClass() - derived = Test.DerivedClass() - - assert base.IsBase() == True - assert derived.IsBase() == False diff --git a/src/tests/test_conversion.py b/src/tests/test_conversion.py index 0ba10a80e..53e5d8051 100644 --- a/src/tests/test_conversion.py +++ b/src/tests/test_conversion.py @@ -2,12 +2,11 @@ """Test CLR <-> Python type conversions.""" -from __future__ import unicode_literals import System import pytest -from Python.Test import ConversionTest, UnicodeString +from Python.Test import ConversionTest -from ._compat import indexbytes, long, unichr, text_type, PY2, PY3 +from ._compat import indexbytes, long, unichr def test_bool_conversion(): @@ -536,14 +535,6 @@ def test_string_conversion(): with pytest.raises(TypeError): ConversionTest().StringField = 1 - - world = UnicodeString() - test_unicode_str = u"안녕" - assert test_unicode_str == text_type(world.value) - assert test_unicode_str == text_type(world.GetString()) - # TODO: not sure what to do for Python 2 here (GH PR #670) - if PY3: - assert test_unicode_str == text_type(world) def test_interface_conversion(): diff --git a/src/tests/test_dictionary.py b/src/tests/test_dictionary.py new file mode 100644 index 000000000..98c3cfb20 --- /dev/null +++ b/src/tests/test_dictionary.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- + +"""Test support for managed dictionaries.""" + +import Python.Test as Test +import System +import pytest + +from ._compat import PY2, UserList, long, range, unichr + + +def test_public_dict(): + """Test public dict.""" + ob = Test.PublicDictionaryTest() + items = ob.items + + assert len(items) == 5 + + assert items['0'] == 0 + assert items['4'] == 4 + + items['0'] = 8 + assert items['0'] == 8 + + items['4'] = 9 + assert items['4'] == 9 + + items['-4'] = 0 + assert items['-4'] == 0 + + items['-1'] = 4 + assert items['-1'] == 4 + +def test_protected_dict(): + """Test protected dict.""" + ob = Test.ProtectedDictionaryTest() + items = ob.items + + assert len(items) == 5 + + assert items['0'] == 0 + assert items['4'] == 4 + + items['0'] = 8 + assert items['0'] == 8 + + items['4'] = 9 + assert items['4'] == 9 + + items['-4'] = 0 + assert items['-4'] == 0 + + items['-1'] = 4 + assert items['-1'] == 4 + +def test_internal_dict(): + """Test internal dict.""" + + with pytest.raises(AttributeError): + ob = Test.InternalDictionaryTest() + _ = ob.items + +def test_private_dict(): + """Test private dict.""" + + with pytest.raises(AttributeError): + ob = Test.PrivateDictionaryTest() + _ = ob.items + +def test_dict_contains(): + """Test dict support for __contains__.""" + + ob = Test.PublicDictionaryTest() + items = ob.items + + assert '0' in items + assert '1' in items + assert '2' in items + assert '3' in items + assert '4' in items + + assert not ('5' in items) + assert not ('-1' in items) + +def test_dict_abuse(): + """Test dict abuse.""" + _class = Test.PublicDictionaryTest + ob = Test.PublicDictionaryTest() + + with pytest.raises(AttributeError): + del _class.__getitem__ + + with pytest.raises(AttributeError): + del ob.__getitem__ + + with pytest.raises(AttributeError): + del _class.__setitem__ + + with pytest.raises(AttributeError): + del ob.__setitem__ + + with pytest.raises(TypeError): + Test.PublicArrayTest.__getitem__(0, 0) + + with pytest.raises(TypeError): + Test.PublicArrayTest.__setitem__(0, 0, 0) + + with pytest.raises(TypeError): + desc = Test.PublicArrayTest.__dict__['__getitem__'] + desc(0, 0) + + with pytest.raises(TypeError): + desc = Test.PublicArrayTest.__dict__['__setitem__'] + desc(0, 0, 0) + +def test_InheritedDictionary(): + """Test class that inherited from IDictionary.""" + items = Test.InheritedDictionaryTest() + + assert len(items) == 5 + + assert items['0'] == 0 + assert items['4'] == 4 + + items['0'] = 8 + assert items['0'] == 8 + + items['4'] = 9 + assert items['4'] == 9 + + items['-4'] = 0 + assert items['-4'] == 0 + + items['-1'] = 4 + assert items['-1'] == 4 + +def test_InheritedDictionary_contains(): + """Test dict support for __contains__ in class that inherited from IDictionary""" + items = Test.InheritedDictionaryTest() + + assert '0' in items + assert '1' in items + assert '2' in items + assert '3' in items + assert '4' in items + + assert not ('5' in items) + assert not ('-1' in items) \ No newline at end of file diff --git a/src/tests/test_exceptions.py b/src/tests/test_exceptions.py index 08b00d77d..c697290ee 100644 --- a/src/tests/test_exceptions.py +++ b/src/tests/test_exceptions.py @@ -289,8 +289,7 @@ def test_python_compat_of_managed_exceptions(): assert e.args == (msg,) assert isinstance(e.args, tuple) if PY3: - strexp = "OverflowException('Simple message" - assert repr(e)[:len(strexp)] == strexp + assert repr(e) == "OverflowException('Simple message',)" elif PY2: assert repr(e) == "OverflowException(u'Simple message',)" diff --git a/src/tests/test_import.py b/src/tests/test_import.py index 25877be15..42cafc4df 100644 --- a/src/tests/test_import.py +++ b/src/tests/test_import.py @@ -3,7 +3,7 @@ """Test the import statement.""" import pytest -import sys + def test_relative_missing_import(): """Test that a relative missing import doesn't crash. @@ -11,12 +11,3 @@ def test_relative_missing_import(): Relative import in the site-packages folder""" with pytest.raises(ImportError): from . import _missing_import - - -def test_import_all_on_second_time(): - """Test import all attributes after a normal import without '*'. - Due to import * only allowed at module level, the test body splited - to a module file.""" - from . import importtest - del sys.modules[importtest.__name__] - diff --git a/src/tests/test_subclass.py b/src/tests/test_subclass.py index ab440d429..43d013c7c 100644 --- a/src/tests/test_subclass.py +++ b/src/tests/test_subclass.py @@ -128,39 +128,6 @@ def test_derived_class(): assert id(x) == id(ob) -def test_derived_traceback(): - """Test python exception traceback in class derived from managed base""" - class DerivedClass(SubClassTest): - __namespace__ = "Python.Test.traceback" - - def foo(self): - print (xyzname) - return None - - import sys,traceback - ob = DerivedClass() - - # direct call - try: - ob.foo() - assert False - except: - e = sys.exc_info() - assert "xyzname" in str(e[1]) - location = traceback.extract_tb(e[2])[-1] - assert location[2] == "foo" - - # call through managed code - try: - FunctionsTest.test_foo(ob) - assert False - except: - e = sys.exc_info() - assert "xyzname" in str(e[1]) - location = traceback.extract_tb(e[2])[-1] - assert location[2] == "foo" - - def test_create_instance(): """Test derived instances can be created from managed code""" DerivedClass = derived_class_fixture(test_create_instance.__name__) diff --git a/tools/geninterop/fake_libc_include/crypt.h b/tools/geninterop/fake_libc_include/crypt.h deleted file mode 100644 index 3b16d481f..000000000 --- a/tools/geninterop/fake_libc_include/crypt.h +++ /dev/null @@ -1 +0,0 @@ -#include "features.h" \ No newline at end of file diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py index f8ef8e561..bf5fdb96b 100644 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -173,8 +173,7 @@ def preprocess_python_headers(): "-D", "__attribute__(x)=", "-D", "__inline__=inline", "-D", "__asm__=;#pragma asm", - "-D", "__int64=long long", - "-D", "_POSIX_THREADS" + "-D", "__int64=long long" ] if hasattr(sys, "abiflags"): @@ -186,7 +185,7 @@ def preprocess_python_headers(): defines.extend(("-D", "PYTHON_WITH_WIDE_UNICODE")) python_h = os.path.join(include_py, "Python.h") - cmd = ["clang", "-pthread", "-I"] + include_dirs + defines + ["-E", python_h] + cmd = ["clang", "-I"] + include_dirs + defines + ["-E", python_h] # normalize as the parser doesn't like windows line endings. lines = []