diff --git a/.bumpversion.cfg b/.bumpversion.cfg deleted file mode 100644 index ac1d31324..000000000 --- a/.bumpversion.cfg +++ /dev/null @@ -1,29 +0,0 @@ -[bumpversion] -current_version = 2.4.0.dev0 -parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? -serialize = - {major}.{minor}.{patch}.{release}{dev} - {major}.{minor}.{patch} - -[bumpversion:part:release] -optional_value = dummy -values = - dev - dummy - -[bumpversion:part:dev] - -[bumpversion:file:setup.py] - -[bumpversion:file:conda.recipe/meta.yaml] - -[bumpversion:file:src/runtime/resources/clr.py] - -[bumpversion:file:src/SharedAssemblyInfo.cs] -serialize = - {major}.{minor}.{patch} - -[bumpversion:file:src/clrmodule/ClrModule.cs] -serialize = - {major}.{minor}.{patch} - diff --git a/.editorconfig b/.editorconfig index 2e7c58ffe..d64f74bc1 100644 --- a/.editorconfig +++ b/.editorconfig @@ -19,6 +19,17 @@ indent_size = 2 [*.{csproj,pyproj,config}] indent_size = 2 +# .NET formatting settings +[*.{cs,vb}] +dotnet_sort_system_directives_first = true +dotnet_separate_import_directive_groups = true + +[*.cs] +csharp_new_line_before_open_brace = all +csharp_new_line_before_else = true +csharp_new_line_before_catch = true +csharp_new_line_before_finally = true + # Solution [*.sln] indent_style = tab diff --git a/.travis.yml b/.travis.yml index 1dadbad1d..2062a35da 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,80 +1,18 @@ -dist: trusty +dist: xenial sudo: false language: python - -matrix: - include: -# --------------------- XPLAT builds ------------------------ - - python: 2.7 - env: &xplat-env - - BUILD_OPTS=--xplat - - NUNIT_PATH=~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe - addons: &xplat-addons - apt: - sources: - - sourceline: deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-trusty-prod trusty main - key_url: https://packages.microsoft.com/keys/microsoft.asc - - sourceline: deb http://download.mono-project.com/repo/ubuntu trusty main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF - packages: - - mono-devel - - ca-certificates-mono - - dotnet-hostfxr-2.0.0 - - dotnet-runtime-2.0.0 - - dotnet-sdk-2.0.0 - - - python: 3.5 - env: *xplat-env - addons: *xplat-addons - - - python: 3.6 - env: *xplat-env - addons: *xplat-addons - - - python: 3.7 - env: *xplat-env - dist: xenial - sudo: true - addons: &xplat-addons-xenial - apt: - sources: - - sourceline: deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod xenial main - key_url: https://packages.microsoft.com/keys/microsoft.asc - - sourceline: deb https://download.mono-project.com/repo/ubuntu stable-xenial main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF - packages: - - mono-devel - - ca-certificates-mono - - dotnet-hostfxr-2.0.0 - - dotnet-runtime-2.0.0 - - dotnet-sdk-2.0.0 - -# --------------------- Classic builds ------------------------ - - python: 2.7 - env: &classic-env - - BUILD_OPTS= - - NUNIT_PATH=./packages/NUnit.*/tools/nunit3-console.exe - - - python: 3.5 - env: *classic-env - - - python: 3.6 - env: *classic-env - - - python: 3.7 - env: *classic-env - dist: xenial - sudo: true - addons: - apt: - sources: - - sourceline: deb http://download.mono-project.com/repo/ubuntu xenial main - key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF - packages: - - mono-devel - - ca-certificates-mono +python: + - 3.8 + - 3.7 + - 3.6 + - 3.5 + - 2.7 env: + matrix: + - BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/ PERF_TESTS_PATH=net461/ + - BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH="" PERF_TESTS_PATH="" + global: - LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so - SEGFAULT_SIGNALS=all @@ -84,11 +22,16 @@ env: addons: apt: sources: - - sourceline: deb http://download.mono-project.com/repo/ubuntu trusty main + - sourceline: deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-xenial-prod xenial main + key_url: https://packages.microsoft.com/keys/microsoft.asc + - sourceline: deb http://download.mono-project.com/repo/ubuntu stable-xenial/snapshots/5.20 main key_url: http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xA6A19B38D3D831EF packages: - mono-devel - ca-certificates-mono + - dotnet-hostfxr-2.2 + - dotnet-runtime-2.2 + - dotnet-sdk-2.2 before_install: # Set-up dll path for embedded tests @@ -102,13 +45,11 @@ install: script: - python -m pytest - - mono $NUNIT_PATH src/embed_tests/bin/Python.EmbeddingTest.dll - - if [[ $BUILD_OPTS == --xplat ]]; then dotnet src/embed_tests/bin/netcoreapp2.0_publish/Python.EmbeddingTest.dll; fi + - $RUN_TESTS src/embed_tests/bin/$EMBED_TESTS_PATH/Python.EmbeddingTest.dll --labels=All + # does not work on Linux, because NuGet package for 2.3 is Windows only + # - "if [[ $TRAVIS_PYTHON_VERSION == '3.5' && $PERF_TESTS_PATH != '' ]]; then mono $NUNIT_PATH src/perf_tests/bin/$PERF_TESTS_PATH/Python.PerformanceTests.dll; fi" after_script: - # Uncomment if need to geninterop, ie. py37 final - # - python tools/geninterop/geninterop.py - # Waiting on mono-coverage, SharpCover or xr.Baboon - coverage xml -i - codecov --file coverage.xml --flags setup_linux diff --git a/AUTHORS.md b/AUTHORS.md index 27aae63f4..5dbf1e1ce 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -2,8 +2,11 @@ ## Development Lead -- Barton Cline ([@BartonCline](https://github.com/BartonCline)) - Benedikt Reinartz ([@filmor](https://github.com/filmor)) +- Victor Milovanov ([@lostmsu](https://github.com/lostmsu)) + +## Former Development Leads +- Barton Cline ([@BartonCline](https://github.com/BartonCline)) - Brian Lloyd ([@brianlloyd](https://github.com/brianlloyd)) - David Anthoff ([@davidanthoff](https://github.com/davidanthoff)) - Denis Akhiyarov ([@denfromufa](https://github.com/denfromufa)) @@ -12,13 +15,17 @@ ## Contributors +- Alex Earl ([@slide](https://github.com/slide)) +- Alex Helms ([@alexhelms](https://github.com/alexhelms)) - Alexandre Catarino([@AlexCatarino](https://github.com/AlexCatarino)) +- Andrey Sant'Anna ([@andreydani](https://github.com/andreydani)) - Arvid JB ([@ArvidJB](https://github.com/ArvidJB)) - Benoît Hudson ([@benoithudson](https://github.com/benoithudson)) - Bradley Friedman ([@leith-bartrich](https://github.com/leith-bartrich)) - Callum Noble ([@callumnoble](https://github.com/callumnoble)) - Christian Heimes ([@tiran](https://github.com/tiran)) - Christoph Gohlke ([@cgohlke](https://github.com/cgohlke)) +- Christopher Bremner ([@chrisjbremner](https://github.com/chrisjbremner)) - Christopher Pow ([@christopherpow](https://github.com/christopherpow)) - Daniel Fernandez ([@fdanny](https://github.com/fdanny)) - Daniel Santana ([@dgsantana](https://github.com/dgsantana)) @@ -26,16 +33,23 @@ - David Lassonde ([@lassond](https://github.com/lassond)) - David Lechner ([@dlech](https://github.com/dlech)) - Dmitriy Se ([@dmitriyse](https://github.com/dmitriyse)) +- Florian Treurniet ([@ftreurni](https://github.com/ftreurni)) - He-chien Tsai ([@t3476](https://github.com/t3476)) --   Ivan Cronyn ([@cronan](https://github.com/cronan)) +- Inna Wiesel ([@inna-w](https://github.com/inna-w)) +- Ivan Cronyn ([@cronan](https://github.com/cronan)) - Jan Krivanek ([@jakrivan](https://github.com/jakrivan)) --   Jeff Reback ([@jreback](https://github.com/jreback)) +- Jeff Reback ([@jreback](https://github.com/jreback)) +- Jeff Robbins ([@jeff17robbins](https://github.com/jeff17robbins)) - Joe Frayne ([@jfrayne](https://github.com/jfrayne)) +- Joe Lidbetter ([@jmlidbetter](https://github.com/jmlidbetter)) +- Joe Savage ([@s4v4g3](https://github.com/s4v4g3)) - John Burnett ([@johnburnett](https://github.com/johnburnett)) - John Wilkes ([@jbw3](https://github.com/jbw3)) - Luke Stratman ([@lstratman](https://github.com/lstratman)) - Konstantin Posudevskiy ([@konstantin-posudevskiy](https://github.com/konstantin-posudevskiy)) - Matthias Dittrich ([@matthid](https://github.com/matthid)) +- Meinrad Recheis ([@henon](https://github.com/henon)) +- Mohamed Koubaa ([@koubaa](https://github.com/koubaa)) - Patrick Stewart ([@patstew](https://github.com/patstew)) - Raphael Nestler ([@rnestler](https://github.com/rnestler)) - Rickard Holmberg ([@rickardraysearch](https://github.com/rickardraysearch)) @@ -43,7 +57,6 @@ - Sean Freitag ([@cowboygneox](https://github.com/cowboygneox)) - Serge Weinstock ([@sweinst](https://github.com/sweinst)) - Simon Mourier ([@smourier](https://github.com/smourier)) -- Victor Milovanov ([@lostmsu](https://github.com/lostmsu)) - Viktoria Kovescses ([@vkovec](https://github.com/vkovec)) - Ville M. Vainio ([@vivainio](https://github.com/vivainio)) - Virgil Dupras ([@hsoft](https://github.com/hsoft)) diff --git a/CHANGELOG.md b/CHANGELOG.md index 289b864cb..ad4016450 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,71 @@ # Changelog -All notable changes to Python for .NET will be documented in this file. -This project adheres to [Semantic Versioning][]. +All notable changes to Python.NET will be documented in this file. This +project adheres to [Semantic Versioning][]. This document follows the conventions laid out in [Keep a CHANGELOG][]. -## [2.4.0][] +## [Unreleased][] + +### Added + +### Changed + +### Fixed + +## [2.5.0][] - 2020-06-14 + +This version improves performance on benchmarks significantly compared to 2.3. + +### Added + +- Automatic NuGet package generation in appveyor and local builds +- Function that sets `Py_NoSiteFlag` to 1. +- Support for Jetson Nano. +- Support for `__len__` for .NET classes that implement ICollection +- `PyExport` attribute to hide .NET types from Python +- `PythonException.Format` method to format exceptions the same as + `traceback.format_exception` +- `Runtime.None` to be able to pass `None` as parameter into Python from .NET +- `PyObject.IsNone()` to check if a Python object is None in .NET. +- Support for Python 3.8 +- Codecs as the designated way to handle automatic conversions between + .NET and Python types + +### Changed + +- Added argument types information to "No method matches given arguments" message +- Moved wheel import in setup.py inside of a try/except to prevent pip collection failures +- Removes `PyLong_GetMax` and `PyClass_New` when targetting Python3 +- Improved performance of calls from Python to C# +- Added support for converting python iterators to C# arrays +- Changed usage of the obsolete function + `GetDelegateForFunctionPointer(IntPtr, Type)` to + `GetDelegateForFunctionPointer(IntPtr)` +- When calling C# from Python, enable passing argument of any type to a + parameter of C# type `object` by wrapping it into `PyObject` instance. + ([#881][i881]) +- Added support for kwarg parameters when calling .NET methods from Python +- Changed method for finding MSBuild using vswhere +- Reworked `Finalizer`. Now objects drop into its queue upon finalization, + which is periodically drained when new objects are created. +- Marked `Runtime.OperatingSystemName` and `Runtime.MachineName` as + `Obsolete`, should never have been `public` in the first place. They also + don't necessarily return a result that matches the `platform` module's. +- Unconditionally depend on `pycparser` for the interop module generation + +### Fixed + +- Fixed runtime that fails loading when using pythonnet in an environment + together with Nuitka +- Fixes bug where delegates get casts (dotnetcore) +- Determine size of interpreter longs at runtime +- Handling exceptions ocurred in ModuleObject's getattribute +- Fill `__classcell__` correctly for Python subclasses of .NET types +- Fixed issue with params methods that are not passed an array. +- Use UTF8 to encode strings passed to `PyRun_String` on Python 3 + +## [2.4.0][] - 2019-05-15 ### Added @@ -24,6 +84,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Added PyObject finalizer support, Python objects referred by C# can be auto collect now ([#692][p692]). - Added detailed comments about aproaches and dangers to handle multi-app-domains ([#625][p625]) - Python 3.7 support, builds and testing added. Defaults changed from Python 3.6 to 3.7 ([#698][p698]) +- Added support for C# types to provide `__repr__` ([#680][p680]) ### Changed @@ -47,7 +108,7 @@ This document follows the conventions laid out in [Keep a CHANGELOG][]. - Fixed conversion of 'float' and 'double' values ([#486][i486]) - Fixed 'clrmethod' for python 2 ([#492][i492]) - Fixed double calling of constructor when deriving from .NET class ([#495][i495]) -- Fixed `clr.GetClrType` when iterating over `System` members ([#607][p607]) +- Fixed `clr.GetClrType` when iterating over `System` members ([#607][p607]) - Fixed `LockRecursionException` when loading assemblies ([#627][i627]) - Fixed errors breaking .NET Remoting on method invoke ([#276][i276]) - Fixed PyObject.GetHashCode ([#676][i676]) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea0f3408e..ffeb792f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,11 +1,15 @@ # How to contribute -PythonNet is developed and maintained by unpaid community members so well +Python.NET is developed and maintained by unpaid community members so well written, documented and tested pull requests are encouraged. By submitting a pull request for this project, you agree to license your contribution under the MIT license to this project. +This project has adopted the code of conduct defined by the Contributor +Covenant to clarify expected behavior in our community. For more information +see the [.NET Foundation Code of Conduct](https://dotnetfoundation.org/code-of-conduct). + ## Getting Started - Make sure you have a [GitHub account](https://github.com/signup/free) @@ -41,3 +45,4 @@ contribution under the MIT license to this project. - [General GitHub documentation](https://help.github.com/) - [GitHub pull request documentation](https://help.github.com/send-pull-requests/) +- [.NET Foundation Code of Conduct](https://dotnetfoundation.org/about/code-of-conduct) diff --git a/LICENSE b/LICENSE index 59abd9c81..19c31a12f 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2006-2019 the contributors of the "Python for .NET" project +Copyright (c) 2006-2020 the contributors of the Python.NET project Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), diff --git a/README.rst b/README.rst index e59ad94f6..55f0e50a1 100644 --- a/README.rst +++ b/README.rst @@ -1,14 +1,14 @@ -pythonnet - Python for .NET +pythonnet - Python.NET =========================== |Join the chat at https://gitter.im/pythonnet/pythonnet| |appveyor shield| |travis shield| |codecov shield| -|license shield| |pypi package version| |python supported shield| +|license shield| |pypi package version| |conda-forge version| |python supported shield| |stackexchange shield| -Python for .NET is a package that gives Python programmers nearly +Python.NET is a package that gives Python programmers nearly seamless integration with the .NET Common Language Runtime (CLR) and provides a powerful application scripting tool for .NET developers. It allows Python code to interact with the CLR, and may also be used to @@ -17,8 +17,7 @@ embed Python into a .NET application. Calling .NET code from Python ----------------------------- -Python for .NET allows CLR namespaces to be treated essentially as -Python packages. +Python.NET allows CLR namespaces to be treated essentially as Python packages. .. code-block:: @@ -90,11 +89,25 @@ Output: int32 [ 6. 10. 12.] + + +Resources +--------- + Information on installation, FAQ, troubleshooting, debugging, and projects using pythonnet can be found in the Wiki: https://github.com/pythonnet/pythonnet/wiki +Mailing list + https://mail.python.org/mailman/listinfo/pythondotnet +Chat + https://gitter.im/pythonnet/pythonnet + +.NET Foundation +--------------- +This project is supported by the `.NET Foundation `_. + .. |Join the chat at https://gitter.im/pythonnet/pythonnet| image:: https://badges.gitter.im/pythonnet/pythonnet.svg :target: https://gitter.im/pythonnet/pythonnet?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge .. |appveyor shield| image:: https://img.shields.io/appveyor/ci/pythonnet/pythonnet/master.svg?label=AppVeyor @@ -111,3 +124,5 @@ https://github.com/pythonnet/pythonnet/wiki :target: https://pypi.python.org/pypi/pythonnet .. |stackexchange shield| image:: https://img.shields.io/badge/StackOverflow-python.net-blue.svg :target: http://stackoverflow.com/questions/tagged/python.net +.. |conda-forge version| image:: https://img.shields.io/conda/vn/conda-forge/pythonnet.svg + :target: https://anaconda.org/conda-forge/pythonnet diff --git a/appveyor.yml b/appveyor.yml index 74b9a9c8e..b58b72372 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -15,24 +15,21 @@ environment: CODECOV_ENV: PYTHON_VERSION, PLATFORM matrix: - - PYTHON_VERSION: 2.7 + - PYTHON_VERSION: 3.8 BUILD_OPTS: --xplat - - PYTHON_VERSION: 3.5 + - PYTHON_VERSION: 3.7 BUILD_OPTS: --xplat - PYTHON_VERSION: 3.6 BUILD_OPTS: --xplat - - PYTHON_VERSION: 3.7 - BUILD_OPTS: --xplat - - PYTHON_VERSION: 2.7 - PYTHON_VERSION: 3.5 - - PYTHON_VERSION: 3.6 + BUILD_OPTS: --xplat + - PYTHON_VERSION: 2.7 + BUILD_OPTS: --xplat + - PYTHON_VERSION: 3.8 - PYTHON_VERSION: 3.7 - -matrix: - allow_failures: - - PYTHON_VERSION: 3.4 - BUILD_OPTS: --xplat - - PYTHON_VERSION: 3.4 + - PYTHON_VERSION: 3.6 + - PYTHON_VERSION: 3.5 + - PYTHON_VERSION: 2.7 init: # Update Environment Variables based on matrix/platform @@ -57,9 +54,8 @@ build_script: - coverage run setup.py bdist_wheel %BUILD_OPTS% test_script: - - pip install --no-index --find-links=.\dist\ pythonnet + - pip install --find-links=.\dist\ pythonnet - ps: .\ci\appveyor_run_tests.ps1 - - ps: .\ci\appveyor_build_recipe.ps1 on_finish: # Temporary disable multiple upload due to codecov limit of 20 per commit. @@ -72,6 +68,7 @@ on_finish: artifacts: - path: dist\* + - path: '.\src\runtime\bin\*.nupkg' notifications: - provider: Slack diff --git a/ci/appveyor_build_recipe.ps1 b/ci/appveyor_build_recipe.ps1 deleted file mode 100644 index 84e0bc7c6..000000000 --- a/ci/appveyor_build_recipe.ps1 +++ /dev/null @@ -1,39 +0,0 @@ -# Build `conda.recipe` only if this is a Pull_Request. Saves time for CI. - -$env:CONDA_PY = "$env:PY_VER" -# Use pre-installed miniconda. Note that location differs if 64bit -$env:CONDA_BLD = "C:\miniconda36" - -if ($env:PLATFORM -eq "x86"){ - $env:CONDA_BLD_ARCH=32 -} else { - $env:CONDA_BLD_ARCH=64 - $env:CONDA_BLD = "$env:CONDA_BLD" + "-x64" -} - -if ($env:APPVEYOR_PULL_REQUEST_NUMBER -or $env:APPVEYOR_REPO_TAG_NAME -or $env:FORCE_CONDA_BUILD -eq "True") { - # Update PATH, and keep a copy to restore at end of this PowerShell script - $old_path = $env:path - $env:path = "$env:CONDA_BLD;$env:CONDA_BLD\Scripts;" + $env:path - - Write-Host "Starting conda install" -ForegroundColor "Green" - conda config --set always_yes True - conda config --set changeps1 False - conda config --set auto_update_conda False - conda install conda-build jinja2 anaconda-client --quiet - conda info - - # why `2>&1 | %{ "$_" }`? Redirect STDERR to STDOUT - # see: http://stackoverflow.com/a/20950421/5208670 - Write-Host "Starting conda build recipe" -ForegroundColor "Green" - conda build conda.recipe --quiet 2>&1 | %{ "$_" } - - $CONDA_PKG=(conda build conda.recipe --output) - Copy-Item $CONDA_PKG .\dist\ - Write-Host "Completed conda build recipe" -ForegroundColor "Green" - - # Restore PATH back to original - $env:path = $old_path -} else { - Write-Host "Skipping conda build recipe" -ForegroundColor "Green" -} diff --git a/ci/appveyor_run_tests.ps1 b/ci/appveyor_run_tests.ps1 index b45440fbe..bd90943d5 100644 --- a/ci/appveyor_run_tests.ps1 +++ b/ci/appveyor_run_tests.ps1 @@ -1,8 +1,13 @@ # Script to simplify AppVeyor configuration and resolve path to tools +$stopwatch = [Diagnostics.Stopwatch]::StartNew() +[array]$timings = @() + # Test Runner framework being used for embedded tests $CS_RUNNER = "nunit3-console" +$XPLAT = $env:BUILD_OPTS -eq "--xplat" + # Needed for ARCH specific runners(NUnit2/XUnit3). Skip for NUnit3 if ($FALSE -and $env:PLATFORM -eq "x86"){ $CS_RUNNER = $CS_RUNNER + "-x86" @@ -11,7 +16,7 @@ if ($FALSE -and $env:PLATFORM -eq "x86"){ # Executable paths for OpenCover # Note if OpenCover fails, it won't affect the exit codes. $OPENCOVER = Resolve-Path .\packages\OpenCover.*\tools\OpenCover.Console.exe -if ($env:BUILD_OPTS -eq "--xplat"){ +if ($XPLAT){ $CS_RUNNER = Resolve-Path $env:USERPROFILE\.nuget\packages\nunit.consolerunner\*\tools\"$CS_RUNNER".exe } else{ @@ -23,6 +28,17 @@ $PY = Get-Command python $CS_TESTS = ".\src\embed_tests\bin\Python.EmbeddingTest.dll" $RUNTIME_DIR = ".\src\runtime\bin\" +function ReportTime { + param([string] $action) + + $timeSpent = $stopwatch.Elapsed + $timings += [pscustomobject]@{action=$action; timeSpent=$timeSpent} + Write-Host $action " in " $timeSpent -ForegroundColor "Green" + $stopwatch.Restart() +} + +ReportTime "Preparation done" + # Run python tests with C# coverage Write-Host ("Starting Python tests") -ForegroundColor "Green" .$OPENCOVER -register:user -searchdirs:"$RUNTIME_DIR" -output:py.coverage ` @@ -31,20 +47,51 @@ Write-Host ("Starting Python tests") -ForegroundColor "Green" $PYTHON_STATUS = $LastExitCode if ($PYTHON_STATUS -ne 0) { Write-Host "Python tests failed, continuing to embedded tests" -ForegroundColor "Red" + ReportTime "" +} else { + ReportTime "Python tests completed" } # Run Embedded tests with C# coverage Write-Host ("Starting embedded tests") -ForegroundColor "Green" .$OPENCOVER -register:user -searchdirs:"$RUNTIME_DIR" -output:cs.coverage ` - -target:"$CS_RUNNER" -targetargs:"$CS_TESTS" ` + -target:"$CS_RUNNER" -targetargs:"$CS_TESTS --labels=All" ` -filter:"+[*]Python.Runtime*" ` -returntargetcode $CS_STATUS = $LastExitCode if ($CS_STATUS -ne 0) { Write-Host "Embedded tests failed" -ForegroundColor "Red" + ReportTime "" +} else { + ReportTime "Embedded tests completed" + + # NuGet for pythonnet-2.3 only has 64-bit binary for Python 3.5 + # the test is only built using modern stack + if (($env:PLATFORM -eq "x64") -and ($XPLAT) -and ($env:PYTHON_VERSION -eq "3.5")) { + # Run C# Performance tests + Write-Host ("Starting performance tests") -ForegroundColor "Green" + if ($XPLAT) { + $CS_PERF_TESTS = ".\src\perf_tests\bin\net461\Python.PerformanceTests.dll" + } + else { + $CS_PERF_TESTS = ".\src\perf_tests\bin\Python.PerformanceTests.dll" + } + &"$CS_RUNNER" "$CS_PERF_TESTS" + $CS_PERF_STATUS = $LastExitCode + if ($CS_PERF_STATUS -ne 0) { + Write-Host "Performance tests (C#) failed" -ForegroundColor "Red" + ReportTime "" + } else { + ReportTime "Performance tests (C#) completed" + } + } else { + Write-Host ("Skipping performance tests for ", $env:PYTHON_VERSION) -ForegroundColor "Yellow" + Write-Host ("on platform ", $env:PLATFORM, " xplat: ", $XPLAT) -ForegroundColor "Yellow" + $CS_PERF_STATUS = 0 + } } -if ($env:BUILD_OPTS -eq "--xplat"){ +if ($XPLAT){ if ($env:PLATFORM -eq "x64") { $DOTNET_CMD = "dotnet" } @@ -54,15 +101,20 @@ if ($env:BUILD_OPTS -eq "--xplat"){ # Run Embedded tests for netcoreapp2.0 (OpenCover currently does not supports dotnet core) Write-Host ("Starting embedded tests for netcoreapp2.0") -ForegroundColor "Green" - &$DOTNET_CMD .\src\embed_tests\bin\netcoreapp2.0_publish\Python.EmbeddingTest.dll + &$DOTNET_CMD ".\src\embed_tests\bin\netcoreapp2.0_publish\Python.EmbeddingTest.dll" $CS_STATUS = $LastExitCode if ($CS_STATUS -ne 0) { Write-Host "Embedded tests for netcoreapp2.0 failed" -ForegroundColor "Red" + ReportTime "" + } else { + ReportTime ".NET Core 2.0 tests completed" } } +Write-Host "Timings:" ($timings | Format-Table | Out-String) -ForegroundColor "Green" + # Set exit code to fail if either Python or Embedded tests failed -if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0) { +if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0 -or $CS_PERF_STATUS -ne 0) { Write-Host "Tests failed" -ForegroundColor "Red" $host.SetShouldExit(1) } diff --git a/conda.recipe/README.md b/conda.recipe/README.md deleted file mode 100644 index 42241999f..000000000 --- a/conda.recipe/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Conda Recipe - -The files here are needed to build Python.Net with conda - -http://conda.pydata.org/docs/building/recipe.html diff --git a/conda.recipe/bld.bat b/conda.recipe/bld.bat deleted file mode 100644 index 27b55f2de..000000000 --- a/conda.recipe/bld.bat +++ /dev/null @@ -1,6 +0,0 @@ -:: build it - -:: set path to modern MSBuild -set PATH=C:\Program Files (x86)\MSBuild\14.0\Bin;C:\Program Files (x86)\MSBuild\15.0\Bin;%PATH% - -%PYTHON% setup.py install diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml deleted file mode 100644 index 545a01268..000000000 --- a/conda.recipe/meta.yaml +++ /dev/null @@ -1,29 +0,0 @@ -package: - name: pythonnet - version: {{ GIT_DESCRIBE_VERSION }} - -build: - skip: True # [not win] - number: {{ GIT_DESCRIBE_NUMBER }} - {% if environ.get('GIT_DESCRIBE_NUMBER', '0') == '0' %}string: py{{ environ.get('PY_VER').replace('.', '') }}_0 - {% else %}string: py{{ environ.get('PY_VER').replace('.', '') }}_{{ environ.get('GIT_BUILD_STR', 'GIT_STUB') }}{% endif %} - -source: - git_url: ../ - -requirements: - build: - - python - - setuptools - - run: - - python - -test: - imports: - - clr - -about: - home: https://github.com/pythonnet/pythonnet - license: MIT - license_file: LICENSE diff --git a/demo/wordpad.py b/demo/wordpad.py index 876372100..409c8ad4d 100644 --- a/demo/wordpad.py +++ b/demo/wordpad.py @@ -382,7 +382,7 @@ def InitializeComponent(self): self.label1.Size = System.Drawing.Size(296, 140) self.label1.TabIndex = 2 self.label1.Text = "Python Wordpad - an example winforms " \ - "application using Python for .NET" + "application using Python.NET" self.AutoScaleBaseSize = System.Drawing.Size(5, 13) self.ClientSize = System.Drawing.Size(300, 150) diff --git a/pythonnet.15.sln b/pythonnet.15.sln index f2015e480..ce863817f 100644 --- a/pythonnet.15.sln +++ b/pythonnet.15.sln @@ -1,189 +1,393 @@ Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.26730.3 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29102.190 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Runtime.15", "src/runtime/Python.Runtime.15.csproj", "{2759F4FF-716B-4828-916F-50FA86613DFC}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Runtime.15", "src\runtime\Python.Runtime.15.csproj", "{2759F4FF-716B-4828-916F-50FA86613DFC}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.EmbeddingTest.15", "src/embed_tests/Python.EmbeddingTest.15.csproj", "{66B8D01A-9906-452A-B09E-BF75EA76468F}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.EmbeddingTest.15", "src\embed_tests\Python.EmbeddingTest.15.csproj", "{66B8D01A-9906-452A-B09E-BF75EA76468F}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "clrmodule.15", "src/clrmodule/clrmodule.15.csproj", "{E08678D4-9A52-4AD5-B63D-8EBC7399981B}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "clrmodule.15", "src\clrmodule\clrmodule.15.csproj", "{E08678D4-9A52-4AD5-B63D-8EBC7399981B}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Console.15", "src/console/Console.15.csproj", "{CDAD305F-8E72-492C-A314-64CF58D472A0}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Console.15", "src\console\Console.15.csproj", "{CDAD305F-8E72-492C-A314-64CF58D472A0}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Test.15", "src/testing/Python.Test.15.csproj", "{F94B547A-E97E-4500-8D53-B4D64D076E5F}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.Test.15", "src\testing\Python.Test.15.csproj", "{F94B547A-E97E-4500-8D53-B4D64D076E5F}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Python.PerformanceTests", "src\perf_tests\Python.PerformanceTests.csproj", "{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repo", "Repo", "{441A0123-F4C6-4EE4-9AEE-315FD79BE2D5}" + ProjectSection(SolutionItems) = preProject + .editorconfig = .editorconfig + .gitignore = .gitignore + CHANGELOG.md = CHANGELOG.md + README.rst = README.rst + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CI", "CI", "{D301657F-5EAF-4534-B280-B858D651B2E5}" + ProjectSection(SolutionItems) = preProject + .travis.yml = .travis.yml + appveyor.yml = appveyor.yml + ci\appveyor_build_recipe.ps1 = ci\appveyor_build_recipe.ps1 + ci\appveyor_run_tests.ps1 = ci\appveyor_run_tests.ps1 + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{57F5D701-F265-4736-A5A2-07249E7A4DA3}" + ProjectSection(SolutionItems) = preProject + setup.py = setup.py + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "conda.recipe", "conda.recipe", "{7FD2404D-0CE8-4645-8DFB-766470E2150E}" + ProjectSection(SolutionItems) = preProject + conda.recipe\bld.bat = conda.recipe\bld.bat + conda.recipe\meta.yaml = conda.recipe\meta.yaml + conda.recipe\README.md = conda.recipe\README.md + EndProjectSection EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + DebugMono|Any CPU = DebugMono|Any CPU DebugMono|x64 = DebugMono|x64 DebugMono|x86 = DebugMono|x86 + DebugMonoPY3|Any CPU = DebugMonoPY3|Any CPU DebugMonoPY3|x64 = DebugMonoPY3|x64 DebugMonoPY3|x86 = DebugMonoPY3|x86 + DebugWin|Any CPU = DebugWin|Any CPU DebugWin|x64 = DebugWin|x64 DebugWin|x86 = DebugWin|x86 + DebugWinPY3|Any CPU = DebugWinPY3|Any CPU DebugWinPY3|x64 = DebugWinPY3|x64 DebugWinPY3|x86 = DebugWinPY3|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + ReleaseMono|Any CPU = ReleaseMono|Any CPU ReleaseMono|x64 = ReleaseMono|x64 ReleaseMono|x86 = ReleaseMono|x86 + ReleaseMonoPY3|Any CPU = ReleaseMonoPY3|Any CPU ReleaseMonoPY3|x64 = ReleaseMonoPY3|x64 ReleaseMonoPY3|x86 = ReleaseMonoPY3|x86 + ReleaseWin|Any CPU = ReleaseWin|Any CPU ReleaseWin|x64 = ReleaseWin|x64 ReleaseWin|x86 = ReleaseWin|x86 + ReleaseWinPY3|Any CPU = ReleaseWinPY3|Any CPU ReleaseWinPY3|x64 = ReleaseWinPY3|x64 ReleaseWinPY3|x86 = ReleaseWinPY3|x86 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|Any CPU.ActiveCfg = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|Any CPU.Build.0 = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|x64.ActiveCfg = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|x64.Build.0 = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|x86.ActiveCfg = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Debug|x86.Build.0 = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|Any CPU.ActiveCfg = DebugMono|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|Any CPU.Build.0 = DebugMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|x64.ActiveCfg = DebugMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|x64.Build.0 = DebugMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|x86.ActiveCfg = DebugMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMono|x86.Build.0 = DebugMono|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|Any CPU.Build.0 = DebugMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|Any CPU.ActiveCfg = DebugWin|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|Any CPU.Build.0 = DebugWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|x64.ActiveCfg = DebugWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|x64.Build.0 = DebugWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|x86.ActiveCfg = DebugWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWin|x86.Build.0 = DebugWin|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|Any CPU.Build.0 = DebugWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|x64.Build.0 = DebugWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.DebugWinPY3|x86.Build.0 = DebugWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|Any CPU.ActiveCfg = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|Any CPU.Build.0 = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|x64.ActiveCfg = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|x64.Build.0 = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|x86.ActiveCfg = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.Release|x86.Build.0 = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|Any CPU.Build.0 = ReleaseMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|x64.ActiveCfg = ReleaseMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|x64.Build.0 = ReleaseMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|x86.ActiveCfg = ReleaseMono|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMono|x86.Build.0 = ReleaseMono|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|Any CPU.Build.0 = ReleaseMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|Any CPU.Build.0 = ReleaseWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|x64.ActiveCfg = ReleaseWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|x64.Build.0 = ReleaseWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|x86.ActiveCfg = ReleaseWin|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWin|x86.Build.0 = ReleaseWin|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|Any CPU + {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|Any CPU.Build.0 = ReleaseWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|Any CPU {2759F4FF-716B-4828-916F-50FA86613DFC}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|Any CPU + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|Any CPU.Build.0 = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|x64.ActiveCfg = DebugWinPY3|x64 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|x64.Build.0 = DebugWinPY3|x64 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|x86.ActiveCfg = DebugWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Debug|x86.Build.0 = DebugWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMono|x64.ActiveCfg = DebugMono|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMono|x64.Build.0 = DebugMono|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMono|x86.ActiveCfg = DebugMono|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMono|x86.Build.0 = DebugMono|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWin|x64.ActiveCfg = DebugWin|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWin|x64.Build.0 = DebugWin|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWin|x86.ActiveCfg = DebugWin|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWin|x86.Build.0 = DebugWin|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|Any CPU.Build.0 = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|x64.ActiveCfg = ReleaseWinPY3|x64 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|x64.Build.0 = ReleaseWinPY3|x64 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|x86.ActiveCfg = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.Release|x86.Build.0 = ReleaseWinPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 + {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {66B8D01A-9906-452A-B09E-BF75EA76468F}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|Any CPU.Build.0 = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|x64.ActiveCfg = DebugWinPY3|x64 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|x64.Build.0 = DebugWinPY3|x64 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|x86.ActiveCfg = DebugWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Debug|x86.Build.0 = DebugWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMono|x64.ActiveCfg = DebugMono|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMono|x86.ActiveCfg = DebugMono|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWin|x64.ActiveCfg = DebugWin|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWin|x64.Build.0 = DebugWin|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWin|x86.ActiveCfg = DebugWin|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWin|x86.Build.0 = DebugWin|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|Any CPU.Build.0 = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|x64.ActiveCfg = ReleaseWinPY3|x64 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|x64.Build.0 = ReleaseWinPY3|x64 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|x86.ActiveCfg = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.Release|x86.Build.0 = ReleaseWinPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 + {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {E08678D4-9A52-4AD5-B63D-8EBC7399981B}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|Any CPU.Build.0 = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|x64.ActiveCfg = DebugWinPY3|x64 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|x64.Build.0 = DebugWinPY3|x64 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|x86.ActiveCfg = DebugWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Debug|x86.Build.0 = DebugWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMono|x64.ActiveCfg = DebugMono|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMono|x64.Build.0 = DebugMono|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMono|x86.ActiveCfg = DebugMono|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMono|x86.Build.0 = DebugMono|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWin|x64.ActiveCfg = DebugWin|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWin|x64.Build.0 = DebugWin|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWin|x86.ActiveCfg = DebugWin|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWin|x86.Build.0 = DebugWin|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|Any CPU.Build.0 = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|x64.ActiveCfg = ReleaseWinPY3|x64 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|x64.Build.0 = ReleaseWinPY3|x64 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|x86.ActiveCfg = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.Release|x86.Build.0 = ReleaseWinPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 + {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {CDAD305F-8E72-492C-A314-64CF58D472A0}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|Any CPU.Build.0 = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|x64.ActiveCfg = DebugWinPY3|x64 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|x64.Build.0 = DebugWinPY3|x64 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|x86.ActiveCfg = DebugWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Debug|x86.Build.0 = DebugWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMono|x64.ActiveCfg = DebugMono|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMono|x64.Build.0 = DebugMono|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMono|x86.ActiveCfg = DebugMono|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMono|x86.Build.0 = DebugMono|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWin|x64.ActiveCfg = DebugWin|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWin|x64.Build.0 = DebugWin|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWin|x86.ActiveCfg = DebugWin|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWin|x86.Build.0 = DebugWin|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|Any CPU.Build.0 = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|x64.ActiveCfg = ReleaseWinPY3|x64 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|x64.Build.0 = ReleaseWinPY3|x64 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|x86.ActiveCfg = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.Release|x86.Build.0 = ReleaseWinPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 + {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 {F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.ActiveCfg = ReleaseWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.Build.0 = ReleaseWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.ActiveCfg = DebugWinPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.Build.0 = DebugWinPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.ActiveCfg = DebugWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.Build.0 = DebugWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x64.ActiveCfg = DebugMono|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x64.Build.0 = DebugMono|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x86.ActiveCfg = DebugMono|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x86.Build.0 = DebugMono|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.ActiveCfg = DebugWin|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.Build.0 = DebugWin|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.ActiveCfg = DebugWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.Build.0 = DebugWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.ActiveCfg = ReleaseWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.Build.0 = ReleaseWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.ActiveCfg = ReleaseWinPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.Build.0 = ReleaseWinPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.ActiveCfg = ReleaseWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.Build.0 = ReleaseWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x64.Build.0 = ReleaseMono|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x86.Build.0 = ReleaseMono|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.Build.0 = ReleaseWin|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.Build.0 = ReleaseWin|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86 + {6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/setup.cfg b/setup.cfg index 38aa3eb3d..19c6f9fc9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,9 +1,3 @@ -# [bumpversion] comments. bumpversion deleted all comments on its file. -# Don't combine `.bumpversion.cfg` with `setup.cfg`. Messes up formatting. -# Don't use `first_value = 1`. It will break `release` bump -# Keep `optional = dummy` needed to bump to release. -# See: https://github.com/peritus/bumpversion/issues/59 - [tool:pytest] xfail_strict = True # -r fsxX: show extra summary info for: (f)ailed, (s)kip, (x)failed, (X)passed diff --git a/setup.py b/setup.py index 7b12997d9..216620211 100644 --- a/setup.py +++ b/setup.py @@ -15,10 +15,14 @@ import sysconfig from distutils import spawn from distutils.command import install, build, build_ext, install_data, install_lib -from wheel import bdist_wheel from setuptools import Extension, setup +try: + from wheel import bdist_wheel +except ImportError: + bdist_wheel = None + # Allow config/verbosity to be set from cli # http://stackoverflow.com/a/4792601/5208670 CONFIG = "Release" # Release or Debug @@ -334,6 +338,7 @@ def build_extension(self, ext): ), '/p:PythonBuildDir="{}"'.format(os.path.abspath(dest_dir)), '/p:PythonInteropFile="{}"'.format(os.path.basename(interop_file)), + "/p:PackageId=pythonnet_py{0}{1}_{2}".format(PY_MAJOR, PY_MINOR, ARCH), "/verbosity:{}".format(VERBOSITY), ] @@ -357,6 +362,16 @@ def build_extension(self, ext): ), shell=use_shell, ) + subprocess.check_call( + " ".join( + cmd + + [ + '"/t:Python_PerformanceTests:publish"', + "/p:TargetFramework=net461", + ] + ), + shell=use_shell, + ) if DEVTOOLS == "Mono" or DEVTOOLS == "dotnet": self._build_monoclr() @@ -383,14 +398,13 @@ def _build_monoclr(self): return raise mono_cflags = _check_output("pkg-config --cflags mono-2", shell=True) - glib_libs = _check_output("pkg-config --libs glib-2.0", shell=True) - glib_cflags = _check_output("pkg-config --cflags glib-2.0", shell=True) - cflags = mono_cflags.strip() + " " + glib_cflags.strip() - libs = mono_libs.strip() + " " + glib_libs.strip() + cflags = mono_cflags.strip() + libs = mono_libs.strip() # build the clr python module clr_ext = Extension( "clr", + language="c++", sources=["src/monoclr/pynetinit.c", "src/monoclr/clrmod.c"], extra_compile_args=cflags.split(" "), extra_link_args=libs.split(" "), @@ -442,26 +456,20 @@ def _find_msbuild_tool(self, tool="msbuild.exe", use_windows_sdk=False): # trying to search path with help of vswhere when MSBuild 15.0 and higher installed. if tool == "msbuild.exe" and use_windows_sdk == False: try: - basePathes = subprocess.check_output( + basePaths = subprocess.check_output( [ "tools\\vswhere\\vswhere.exe", "-latest", "-version", - "[15.0, 16.0)", + "[15.0,)", "-requires", "Microsoft.Component.MSBuild", - "-property", - "InstallationPath", + "-find", + "MSBuild\**\Bin\MSBuild.exe", ] ).splitlines() - if len(basePathes): - return os.path.join( - basePathes[0].decode(sys.stdout.encoding or "utf-8"), - "MSBuild", - "15.0", - "Bin", - "MSBuild.exe", - ) + if len(basePaths): + return basePaths[0].decode(sys.stdout.encoding or "utf-8") except: pass # keep trying to search by old method. @@ -513,26 +521,20 @@ def _find_msbuild_tool(self, tool="msbuild.exe", use_windows_sdk=False): def _find_msbuild_tool_15(self): """Return full path to one of the Microsoft build tools""" try: - basePathes = subprocess.check_output( + basePaths = subprocess.check_output( [ "tools\\vswhere\\vswhere.exe", "-latest", "-version", - "[15.0, 16.0)", + "[15.0,)", "-requires", "Microsoft.Component.MSBuild", - "-property", - "InstallationPath", + "-find", + "MSBuild\**\Bin\MSBuild.exe", ] ).splitlines() - if len(basePathes): - return os.path.join( - basePathes[0].decode(sys.stdout.encoding or "utf-8"), - "MSBuild", - "15.0", - "Bin", - "MSBuild.exe", - ) + if len(basePaths): + return basePaths[0].decode(sys.stdout.encoding or "utf-8") else: raise RuntimeError("MSBuild >=15.0 could not be found.") except subprocess.CalledProcessError as e: @@ -593,21 +595,21 @@ def run(self): _update_xlat_devtools() return install.install.run(self) +if bdist_wheel: + class BDistWheelPythonnet(bdist_wheel.bdist_wheel): + user_options = bdist_wheel.bdist_wheel.user_options + [("xplat", None, None)] -class BDistWheelPythonnet(bdist_wheel.bdist_wheel): - user_options = bdist_wheel.bdist_wheel.user_options + [("xplat", None, None)] - - def initialize_options(self): - bdist_wheel.bdist_wheel.initialize_options(self) - self.xplat = None + def initialize_options(self): + bdist_wheel.bdist_wheel.initialize_options(self) + self.xplat = None - def finalize_options(self): - bdist_wheel.bdist_wheel.finalize_options(self) + def finalize_options(self): + bdist_wheel.bdist_wheel.finalize_options(self) - def run(self): - if self.xplat: - _update_xlat_devtools() - return bdist_wheel.bdist_wheel.run(self) + def run(self): + if self.xplat: + _update_xlat_devtools() + return bdist_wheel.bdist_wheel.run(self) ############################################################################### @@ -616,29 +618,28 @@ def run(self): if setupdir: os.chdir(setupdir) -setup_requires = [] -if not os.path.exists(_get_interop_filename()): - setup_requires.append("pycparser") +cmdclass={ + "install": InstallPythonnet, + "build_ext": BuildExtPythonnet, + "install_lib": InstallLibPythonnet, + "install_data": InstallDataPythonnet, +} +if bdist_wheel: + cmdclass["bdist_wheel"] = BDistWheelPythonnet setup( name="pythonnet", - version="2.4.0", + version="2.5.0", description=".Net and Mono integration for Python", url="https://pythonnet.github.io/", license="MIT", - author="The Python for .Net developers", - author_email="pythondotnet@python.org", - setup_requires=setup_requires, + author="The Contributors of the Python.NET Project", + author_email="pythonnet@python.org", + install_requires=["pycparser"], long_description=_get_long_description(), ext_modules=[Extension("clr", sources=list(_get_source_files()))], data_files=[("{install_platlib}", ["{build_lib}/Python.Runtime.dll"])], - cmdclass={ - "install": InstallPythonnet, - "build_ext": BuildExtPythonnet, - "install_lib": InstallLibPythonnet, - "install_data": InstallDataPythonnet, - "bdist_wheel": BDistWheelPythonnet, - }, + cmdclass=cmdclass, classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", @@ -647,10 +648,10 @@ def run(self): "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", "Operating System :: MacOS :: MacOS X", diff --git a/src/SharedAssemblyInfo.cs b/src/SharedAssemblyInfo.cs index 669a43746..dd057b144 100644 --- a/src/SharedAssemblyInfo.cs +++ b/src/SharedAssemblyInfo.cs @@ -8,8 +8,8 @@ // associated with an assembly. [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("pythonnet")] -[assembly: AssemblyProduct("Python for .NET")] -[assembly: AssemblyCopyright("Copyright (c) 2006-2019 the contributors of the 'Python for .NET' project")] +[assembly: AssemblyProduct("Python.NET")] +[assembly: AssemblyCopyright("Copyright (c) 2006-2020 the contributors of the Python.NET project")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] @@ -25,4 +25,4 @@ // Version Information. Keeping it simple. May need to revisit for Nuget // See: https://codingforsmarties.wordpress.com/2016/01/21/how-to-version-assemblies-destined-for-nuget/ // AssemblyVersion can only be numeric -[assembly: AssemblyVersion("2.4.0")] +[assembly: AssemblyVersion("2.5.0")] diff --git a/src/clrmodule/ClrModule.cs b/src/clrmodule/ClrModule.cs index 3bb3a533b..377be66d1 100644 --- a/src/clrmodule/ClrModule.cs +++ b/src/clrmodule/ClrModule.cs @@ -53,7 +53,7 @@ public static void initclr() { #if USE_PYTHON_RUNTIME_VERSION // Has no effect until SNK works. Keep updated anyways. - Version = new Version("2.4.0"), + Version = new Version("2.5.0"), #endif CultureInfo = CultureInfo.InvariantCulture }; diff --git a/src/clrmodule/clrmodule.15.csproj b/src/clrmodule/clrmodule.15.csproj index 2585ffdd2..7fc9c2dda 100644 --- a/src/clrmodule/clrmodule.15.csproj +++ b/src/clrmodule/clrmodule.15.csproj @@ -9,7 +9,7 @@ clrmodule clrmodule clrmodule - 2.4.0 + 2.5.0 false false false diff --git a/src/console/Console.15.csproj b/src/console/Console.15.csproj index ec5008036..3c51caa31 100644 --- a/src/console/Console.15.csproj +++ b/src/console/Console.15.csproj @@ -8,7 +8,7 @@ nPython Python.Runtime nPython - 2.4.0 + 2.5.0 false false false diff --git a/src/embed_tests/CodecGroups.cs b/src/embed_tests/CodecGroups.cs new file mode 100644 index 000000000..5dd40210f --- /dev/null +++ b/src/embed_tests/CodecGroups.cs @@ -0,0 +1,132 @@ +namespace Python.EmbeddingTest +{ + using System; + using System.Linq; + using NUnit.Framework; + using Python.Runtime; + using Python.Runtime.Codecs; + + public class CodecGroups + { + [Test] + public void GetEncodersByType() + { + var encoder1 = new ObjectToEncoderInstanceEncoder(); + var encoder2 = new ObjectToEncoderInstanceEncoder(); + var group = new EncoderGroup { + new ObjectToEncoderInstanceEncoder>(), + encoder1, + encoder2, + }; + + var got = group.GetEncoders(typeof(Uri)).ToArray(); + CollectionAssert.AreEqual(new[]{encoder1, encoder2}, got); + } + + [Test] + public void CanEncode() + { + var group = new EncoderGroup { + new ObjectToEncoderInstanceEncoder>(), + new ObjectToEncoderInstanceEncoder(), + }; + + Assert.IsTrue(group.CanEncode(typeof(Tuple))); + Assert.IsTrue(group.CanEncode(typeof(Uri))); + Assert.IsFalse(group.CanEncode(typeof(string))); + } + + [Test] + public void Encodes() + { + var encoder0 = new ObjectToEncoderInstanceEncoder>(); + var encoder1 = new ObjectToEncoderInstanceEncoder(); + var encoder2 = new ObjectToEncoderInstanceEncoder(); + var group = new EncoderGroup { + encoder0, + encoder1, + encoder2, + }; + + var uri = group.TryEncode(new Uri("data:")); + var clrObject = (CLRObject)ManagedType.GetManagedObject(uri.Handle); + Assert.AreSame(encoder1, clrObject.inst); + Assert.AreNotSame(encoder2, clrObject.inst); + + var tuple = group.TryEncode(Tuple.Create(1)); + clrObject = (CLRObject)ManagedType.GetManagedObject(tuple.Handle); + Assert.AreSame(encoder0, clrObject.inst); + } + + [Test] + public void GetDecodersByTypes() + { + var pyint = new PyInt(10).GetPythonType(); + var pyfloat = new PyFloat(10).GetPythonType(); + var pystr = new PyString("world").GetPythonType(); + var decoder1 = new DecoderReturningPredefinedValue(pyint, decodeResult: 42); + var decoder2 = new DecoderReturningPredefinedValue(pyfloat, decodeResult: "atad:"); + var group = new DecoderGroup { + decoder1, + decoder2, + }; + + var decoder = group.GetDecoder(pyfloat, typeof(string)); + Assert.AreSame(decoder2, decoder); + decoder = group.GetDecoder(pystr, typeof(string)); + Assert.IsNull(decoder); + decoder = group.GetDecoder(pyint, typeof(long)); + Assert.AreSame(decoder1, decoder); + } + [Test] + public void CanDecode() + { + var pyint = new PyInt(10).GetPythonType(); + var pyfloat = new PyFloat(10).GetPythonType(); + var pystr = new PyString("world").GetPythonType(); + var decoder1 = new DecoderReturningPredefinedValue(pyint, decodeResult: 42); + var decoder2 = new DecoderReturningPredefinedValue(pyfloat, decodeResult: "atad:"); + var group = new DecoderGroup { + decoder1, + decoder2, + }; + + Assert.IsTrue(group.CanDecode(pyint, typeof(long))); + Assert.IsFalse(group.CanDecode(pyint, typeof(int))); + Assert.IsTrue(group.CanDecode(pyfloat, typeof(string))); + Assert.IsFalse(group.CanDecode(pystr, typeof(string))); + } + + [Test] + public void Decodes() + { + var pyint = new PyInt(10).GetPythonType(); + var pyfloat = new PyFloat(10).GetPythonType(); + var decoder1 = new DecoderReturningPredefinedValue(pyint, decodeResult: 42); + var decoder2 = new DecoderReturningPredefinedValue(pyfloat, decodeResult: "atad:"); + var group = new DecoderGroup { + decoder1, + decoder2, + }; + + Assert.IsTrue(group.TryDecode(new PyInt(10), out long longResult)); + Assert.AreEqual(42, longResult); + Assert.IsTrue(group.TryDecode(new PyFloat(10), out string strResult)); + Assert.AreSame("atad:", strResult); + + Assert.IsFalse(group.TryDecode(new PyInt(10), out int _)); + } + + [SetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [TearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + } +} diff --git a/src/embed_tests/Codecs.cs b/src/embed_tests/Codecs.cs new file mode 100644 index 000000000..18fcd32d1 --- /dev/null +++ b/src/embed_tests/Codecs.cs @@ -0,0 +1,123 @@ +namespace Python.EmbeddingTest { + using System; + using System.Collections.Generic; + using System.Text; + using NUnit.Framework; + using Python.Runtime; + using Python.Runtime.Codecs; + + public class Codecs { + [SetUp] + public void SetUp() { + PythonEngine.Initialize(); + } + + [TearDown] + public void Dispose() { + PythonEngine.Shutdown(); + } + + [Test] + public void ConversionsGeneric() { + ConversionsGeneric, ValueTuple>(); + } + + static void ConversionsGeneric() { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + T restored = default; + using (Py.GIL()) + using (var scope = Py.CreateScope()) { + void Accept(T value) => restored = value; + var accept = new Action(Accept).ToPython(); + scope.Set(nameof(tuple), tuple); + scope.Set(nameof(accept), accept); + scope.Exec($"{nameof(accept)}({nameof(tuple)})"); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void ConversionsObject() { + ConversionsObject, ValueTuple>(); + } + static void ConversionsObject() { + TupleCodec.Register(); + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + T restored = default; + using (Py.GIL()) + using (var scope = Py.CreateScope()) { + void Accept(object value) => restored = (T)value; + var accept = new Action(Accept).ToPython(); + scope.Set(nameof(tuple), tuple); + scope.Set(nameof(accept), accept); + scope.Exec($"{nameof(accept)}({nameof(tuple)})"); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void TupleRoundtripObject() { + TupleRoundtripObject, ValueTuple>(); + } + static void TupleRoundtripObject() { + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + using (Py.GIL()) { + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out object restored)); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + + [Test] + public void TupleRoundtripGeneric() { + TupleRoundtripGeneric, ValueTuple>(); + } + + static void TupleRoundtripGeneric() { + var tuple = Activator.CreateInstance(typeof(T), 42, "42", new object()); + using (Py.GIL()) { + var pyTuple = TupleCodec.Instance.TryEncode(tuple); + Assert.IsTrue(TupleCodec.Instance.TryDecode(pyTuple, out T restored)); + Assert.AreEqual(expected: tuple, actual: restored); + } + } + } + + /// + /// "Decodes" only objects of exact type . + /// Result is just the raw proxy to the encoder instance itself. + /// + class ObjectToEncoderInstanceEncoder : IPyObjectEncoder + { + public bool CanEncode(Type type) => type == typeof(T); + public PyObject TryEncode(object value) => PyObject.FromManagedObject(this); + } + + /// + /// Decodes object of specified Python type to the predefined value + /// + /// Type of the + class DecoderReturningPredefinedValue : IPyObjectDecoder + { + public PyObject TheOnlySupportedSourceType { get; } + public TTarget DecodeResult { get; } + + public DecoderReturningPredefinedValue(PyObject objectType, TTarget decodeResult) + { + this.TheOnlySupportedSourceType = objectType; + this.DecodeResult = decodeResult; + } + + public bool CanDecode(PyObject objectType, Type targetType) + => objectType.Handle == TheOnlySupportedSourceType.Handle + && targetType == typeof(TTarget); + public bool TryDecode(PyObject pyObj, out T value) + { + if (typeof(T) != typeof(TTarget)) + throw new ArgumentException(nameof(T)); + value = (T)(object)DecodeResult; + return true; + } + } +} diff --git a/src/embed_tests/Python.EmbeddingTest.15.csproj b/src/embed_tests/Python.EmbeddingTest.15.csproj index a741a589e..eb6957656 100644 --- a/src/embed_tests/Python.EmbeddingTest.15.csproj +++ b/src/embed_tests/Python.EmbeddingTest.15.csproj @@ -10,7 +10,7 @@ Python.EmbeddingTest Python.EmbeddingTest Python.EmbeddingTest - 2.4.0 + 2.5.0 false false false @@ -23,7 +23,7 @@ ..\..\ $(SolutionDir)\bin\ $(OutputPath)\$(TargetFramework)_publish - 6 + 7.3 prompt $(PYTHONNET_DEFINE_CONSTANTS) XPLAT @@ -77,13 +77,18 @@ - - - - + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + - + diff --git a/src/embed_tests/Python.EmbeddingTest.csproj b/src/embed_tests/Python.EmbeddingTest.csproj index faa55fa27..5dee66e64 100644 --- a/src/embed_tests/Python.EmbeddingTest.csproj +++ b/src/embed_tests/Python.EmbeddingTest.csproj @@ -14,7 +14,7 @@ 1591 ..\..\ $(SolutionDir)\bin\ - 6 + 7.3 true prompt @@ -70,8 +70,11 @@ - - ..\..\packages\NUnit.3.7.1\lib\net40\nunit.framework.dll + + ..\..\packages\NUnit.3.12.0\lib\net40\nunit.framework.dll + + + ..\..\packages\System.ValueTuple.4.5.0\lib\portable-net40+sl4+win8+wp8\System.ValueTuple.dll @@ -80,15 +83,19 @@ + + + + @@ -126,4 +133,10 @@ + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + diff --git a/src/embed_tests/References.cs b/src/embed_tests/References.cs new file mode 100644 index 000000000..1d29e85c7 --- /dev/null +++ b/src/embed_tests/References.cs @@ -0,0 +1,55 @@ +namespace Python.EmbeddingTest +{ + using NUnit.Framework; + using Python.Runtime; + + public class References + { + private Py.GILState _gs; + + [SetUp] + public void SetUp() + { + _gs = Py.GIL(); + } + + [TearDown] + public void Dispose() + { + _gs.Dispose(); + } + + [Test] + public void MoveToPyObject_SetsNull() + { + var dict = new PyDict(); + NewReference reference = Runtime.PyDict_Items(dict.Handle); + try + { + Assert.IsFalse(reference.IsNull()); + + using (reference.MoveToPyObject()) + Assert.IsTrue(reference.IsNull()); + } + finally + { + reference.Dispose(); + } + } + + [Test] + public void CanBorrowFromNewReference() + { + var dict = new PyDict(); + NewReference reference = Runtime.PyDict_Items(dict.Handle); + try + { + PythonException.ThrowIfIsNotZero(Runtime.PyList_Reverse(reference)); + } + finally + { + reference.Dispose(); + } + } + } +} diff --git a/src/embed_tests/TestCallbacks.cs b/src/embed_tests/TestCallbacks.cs new file mode 100644 index 000000000..220b0a86a --- /dev/null +++ b/src/embed_tests/TestCallbacks.cs @@ -0,0 +1,35 @@ +using System; + +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest { + using Runtime = Python.Runtime.Runtime; + + public class TestCallbacks { + [OneTimeSetUp] + public void SetUp() { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() { + PythonEngine.Shutdown(); + } + + [Test] + public void TestNoOverloadException() { + int passed = 0; + var aFunctionThatCallsIntoPython = new Action(value => passed = value); + using (Py.GIL()) { + dynamic callWith42 = PythonEngine.Eval("lambda f: f([42])"); + var error = Assert.Throws(() => callWith42(aFunctionThatCallsIntoPython.ToPython())); + Assert.AreEqual("TypeError", error.PythonTypeName); + string expectedArgTypes = Runtime.IsPython2 + ? "()" + : "()"; + StringAssert.EndsWith(expectedArgTypes, error.Message); + } + } + } +} diff --git a/src/embed_tests/TestConverter.cs b/src/embed_tests/TestConverter.cs index caaec311b..40219973b 100644 --- a/src/embed_tests/TestConverter.cs +++ b/src/embed_tests/TestConverter.cs @@ -1,3 +1,5 @@ +using System; +using System.Collections.Generic; using NUnit.Framework; using Python.Runtime; @@ -44,5 +46,26 @@ public void TestConvertDoubleToManaged( Assert.IsTrue(converted); Assert.IsTrue(((double) convertedValue).Equals(testValue)); } + + [Test] + public void RawListProxy() + { + var list = new List {"hello", "world"}; + var listProxy = PyObject.FromManagedObject(list); + var clrObject = (CLRObject)ManagedType.GetManagedObject(listProxy.Handle); + Assert.AreSame(list, clrObject.inst); + } + + [Test] + public void RawPyObjectProxy() + { + var pyObject = "hello world!".ToPython(); + var pyObjectProxy = PyObject.FromManagedObject(pyObject); + var clrObject = (CLRObject)ManagedType.GetManagedObject(pyObjectProxy.Handle); + Assert.AreSame(pyObject, clrObject.inst); + + var proxiedHandle = pyObjectProxy.GetAttr("Handle").As(); + Assert.AreEqual(pyObject.Handle, proxiedHandle); + } } } diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 53838f315..f82767af1 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -45,7 +45,7 @@ public void CollectBasicObject() called = true; }; - Assert.IsFalse(called); + Assert.IsFalse(called, "The event handler was called before it was installed"); Finalizer.Instance.CollectOnce += handler; WeakReference shortWeak; @@ -55,23 +55,35 @@ public void CollectBasicObject() } FullGCCollect(); // The object has been resurrected - Assert.IsFalse(shortWeak.IsAlive); - Assert.IsTrue(longWeak.IsAlive); + Warn.If( + shortWeak.IsAlive, + "The referenced object is alive although it should have been collected", + shortWeak + ); + Assert.IsTrue( + longWeak.IsAlive, + "The reference object is not alive although it should still be", + longWeak + ); { var garbage = Finalizer.Instance.GetCollectedObjects(); - Assert.NotZero(garbage.Count); - Assert.IsTrue(garbage.Any(T => ReferenceEquals(T.Target, longWeak.Target))); + Assert.NotZero(garbage.Count, "There should still be garbage around"); + Warn.Unless( + garbage.Any(T => ReferenceEquals(T.Target, longWeak.Target)), + $"The {nameof(longWeak)} reference doesn't show up in the garbage list", + garbage + ); } try { - Finalizer.Instance.Collect(forceDispose: false); + Finalizer.Instance.Collect(); } finally { Finalizer.Instance.CollectOnce -= handler; } - Assert.IsTrue(called); + Assert.IsTrue(called, "The event handler was not called during finalization"); Assert.GreaterOrEqual(objectCount, 1); } diff --git a/src/embed_tests/TestGILState.cs b/src/embed_tests/TestGILState.cs new file mode 100644 index 000000000..bf6f02dc6 --- /dev/null +++ b/src/embed_tests/TestGILState.cs @@ -0,0 +1,33 @@ +namespace Python.EmbeddingTest +{ + using NUnit.Framework; + using Python.Runtime; + + public class TestGILState + { + /// + /// Ensure, that calling multiple times is safe + /// + [Test] + public void CanDisposeMultipleTimes() + { + using (var gilState = Py.GIL()) + { + for(int i = 0; i < 50; i++) + gilState.Dispose(); + } + } + + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + } +} diff --git a/src/embed_tests/TestInstanceWrapping.cs b/src/embed_tests/TestInstanceWrapping.cs new file mode 100644 index 000000000..8be207c00 --- /dev/null +++ b/src/embed_tests/TestInstanceWrapping.cs @@ -0,0 +1,58 @@ +using System; +using System.Globalization; +using NUnit.Framework; +using Python.Runtime; + +namespace Python.EmbeddingTest +{ + public class TestInstanceWrapping + { + [OneTimeSetUp] + public void SetUp() + { + PythonEngine.Initialize(); + } + + [OneTimeTearDown] + public void Dispose() + { + PythonEngine.Shutdown(); + } + + // regression test for https://github.com/pythonnet/pythonnet/issues/811 + [Test] + public void OverloadResolution_UnknownToObject() + { + var overloaded = new Overloaded(); + using (Py.GIL()) + { + var o = overloaded.ToPython(); + + dynamic callWithSelf = PythonEngine.Eval("lambda o: o.ObjOrClass(object())"); + callWithSelf(o); + Assert.AreEqual(Overloaded.Object, overloaded.Value); + } + } + + class Base {} + class Derived: Base { } + + class Overloaded: Derived + { + public int Value { get; set; } + public void IntOrStr(int arg) => this.Value = arg; + public void IntOrStr(string arg) => + this.Value = int.Parse(arg, NumberStyles.Integer, CultureInfo.InvariantCulture); + + public const int Object = 1; + public const int ConcreteClass = 2; + public void ObjOrClass(object _) => this.Value = Object; + public void ObjOrClass(Overloaded _) => this.Value = ConcreteClass; + + public const int Base = ConcreteClass + 1; + public const int Derived = Base + 1; + public void BaseOrDerived(Base _) => this.Value = Base; + public void BaseOrDerived(Derived _) => this.Value = Derived; + } + } +} diff --git a/src/embed_tests/TestPyObject.cs b/src/embed_tests/TestPyObject.cs index 65ac20e9a..d4952d4a3 100644 --- a/src/embed_tests/TestPyObject.cs +++ b/src/embed_tests/TestPyObject.cs @@ -57,5 +57,11 @@ def add(self, x, y): Assert.IsTrue(memberNames.Contains(expectedName), "Could not find member '{0}'.", expectedName); } } + + [Test] + public void InvokeNull() { + var list = PythonEngine.Eval("list"); + Assert.Throws(() => list.Invoke(new PyObject[] {null})); + } } } diff --git a/src/embed_tests/TestPyScope.cs b/src/embed_tests/TestPyScope.cs index 21c0d2b3f..701e698ec 100644 --- a/src/embed_tests/TestPyScope.cs +++ b/src/embed_tests/TestPyScope.cs @@ -1,4 +1,5 @@ using System; +using System.Threading; using NUnit.Framework; using Python.Runtime; @@ -337,13 +338,16 @@ public void TestThread() //add function to the scope //can be call many times, more efficient than ast ps.Exec( + "import threading\n" + + "lock = threading.Lock()\n" + "def update():\n" + - " global res, th_cnt\n" + + " global res, th_cnt\n" + + " with lock:\n" + " res += bb + 1\n" + " th_cnt += 1\n" ); } - int th_cnt = 3; + int th_cnt = 100; for (int i = 0; i < th_cnt; i++) { System.Threading.Thread th = new System.Threading.Thread(() => @@ -364,7 +368,7 @@ public void TestThread() { cnt = ps.Get("th_cnt"); } - System.Threading.Thread.Sleep(10); + Thread.Yield(); } using (Py.GIL()) { diff --git a/src/embed_tests/TestPythonException.cs b/src/embed_tests/TestPythonException.cs index 57a8d54af..000c32ca3 100644 --- a/src/embed_tests/TestPythonException.cs +++ b/src/embed_tests/TestPythonException.cs @@ -54,5 +54,41 @@ public void TestPythonErrorTypeName() Assert.That(ex.PythonTypeName, Is.EqualTo("ModuleNotFoundError").Or.EqualTo("ImportError")); } } + + [Test] + public void TestPythonExceptionFormat() + { + try + { + PythonEngine.Exec("raise ValueError('Error!')"); + Assert.Fail("Exception should have been raised"); + } + catch (PythonException ex) + { + Assert.That(ex.Format(), Does.Contain("Traceback").And.Contains("(most recent call last):").And.Contains("ValueError: Error!")); + } + } + + [Test] + public void TestPythonExceptionFormatNoError() + { + var ex = new PythonException(); + Assert.AreEqual(ex.StackTrace, ex.Format()); + } + + [Test] + public void TestPythonExceptionFormatNoTraceback() + { + try + { + var module = PythonEngine.ImportModule("really____unknown___module"); + Assert.Fail("Unknown module should not be loaded"); + } + catch (PythonException ex) + { + // ImportError/ModuleNotFoundError do not have a traceback when not running in a script + Assert.AreEqual(ex.StackTrace, ex.Format()); + } + } } } diff --git a/src/embed_tests/TestRuntime.cs b/src/embed_tests/TestRuntime.cs index ac1fa1ac0..4129d3df3 100644 --- a/src/embed_tests/TestRuntime.cs +++ b/src/embed_tests/TestRuntime.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using NUnit.Framework; using Python.Runtime; +using Python.Runtime.Platform; namespace Python.EmbeddingTest { @@ -26,11 +28,8 @@ public static void PlatformCache() { Runtime.Runtime.Initialize(); - Assert.That(Runtime.Runtime.Machine, Is.Not.EqualTo(Runtime.Runtime.MachineType.Other)); - Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.MachineName)); - - Assert.That(Runtime.Runtime.OperatingSystem, Is.Not.EqualTo(Runtime.Runtime.OperatingSystemType.Other)); - Assert.That(!string.IsNullOrEmpty(Runtime.Runtime.OperatingSystemName)); + Assert.That(Runtime.Runtime.Machine, Is.Not.EqualTo(MachineType.Other)); + Assert.That(Runtime.Runtime.OperatingSystem, Is.Not.EqualTo(OperatingSystemType.Other)); // Don't shut down the runtime: if the python engine was initialized // but not shut down by another test, we'd end up in a bad state. @@ -39,7 +38,7 @@ public static void PlatformCache() [Test] public static void Py_IsInitializedValue() { - Runtime.Runtime.Py_Finalize(); + Runtime.Runtime.Py_Finalize(); Assert.AreEqual(0, Runtime.Runtime.Py_IsInitialized()); Runtime.Runtime.Py_Initialize(); Assert.AreEqual(1, Runtime.Runtime.Py_IsInitialized()); @@ -109,9 +108,15 @@ public static void PyCheck_Iter_PyObject_IsIterable_ThreadingLock_Test() // Create an instance of threading.Lock, which is one of the very few types that does not have the // TypeFlags.HaveIter set in Python 2. This tests a different code path in PyObject_IsIterable and PyIter_Check. var threading = Runtime.Runtime.PyImport_ImportModule("threading"); + Exceptions.ErrorCheck(threading); var threadingDict = Runtime.Runtime.PyModule_GetDict(threading); + Exceptions.ErrorCheck(threadingDict); var lockType = Runtime.Runtime.PyDict_GetItemString(threadingDict, "Lock"); + if (lockType == IntPtr.Zero) + throw new KeyNotFoundException("class 'Lock' was not found in 'threading'"); + var lockInstance = Runtime.Runtime.PyObject_CallObject(lockType, Runtime.Runtime.PyTuple_New(0)); + Exceptions.ErrorCheck(lockInstance); Assert.IsFalse(Runtime.Runtime.PyObject_IsIterable(lockInstance)); Assert.IsFalse(Runtime.Runtime.PyIter_Check(lockInstance)); diff --git a/src/embed_tests/packages.config b/src/embed_tests/packages.config index 8c175f441..590eaef8c 100644 --- a/src/embed_tests/packages.config +++ b/src/embed_tests/packages.config @@ -1,5 +1,6 @@ - + - - + + + diff --git a/src/embed_tests/pyimport.cs b/src/embed_tests/pyimport.cs index acb3433de..6b2408745 100644 --- a/src/embed_tests/pyimport.cs +++ b/src/embed_tests/pyimport.cs @@ -39,7 +39,7 @@ public void SetUp() IntPtr str = Runtime.Runtime.PyString_FromString(testPath); IntPtr path = Runtime.Runtime.PySys_GetObject("path"); - Runtime.Runtime.PyList_Append(path, str); + Runtime.Runtime.PyList_Append(new BorrowedReference(path), str); } [TearDown] diff --git a/src/monoclr/pynetclr.h b/src/monoclr/pynetclr.h index c5e181156..1863b1d43 100644 --- a/src/monoclr/pynetclr.h +++ b/src/monoclr/pynetclr.h @@ -7,12 +7,15 @@ #include #include #include -#include #define MONO_VERSION "v4.0.30319.1" #define MONO_DOMAIN "Python.Runtime" #define PR_ASSEMBLY "Python.Runtime.dll" +#ifdef __cplusplus +extern "C" { +#endif + typedef struct { MonoDomain *domain; @@ -27,7 +30,11 @@ typedef struct PyNet_Args *PyNet_Init(int); void PyNet_Finalize(PyNet_Args *); -void main_thread_handler(gpointer user_data); +void main_thread_handler(PyNet_Args *user_data); char *PyNet_ExceptionToString(MonoObject *); +#ifdef __cplusplus +} +#endif + #endif // PYNET_CLR_H diff --git a/src/monoclr/pynetinit.c b/src/monoclr/pynetinit.c index 8b49ddae3..149d52296 100644 --- a/src/monoclr/pynetinit.c +++ b/src/monoclr/pynetinit.c @@ -19,6 +19,15 @@ PyNet_Args *PyNet_Init(int ext) pn_args->shutdown = NULL; pn_args->module = NULL; +#ifdef __linux__ + // Force preload libmono-2.0 as global. Without this, on some systems + // symbols from libmono are not found by libmononative (which implements + // some of the System.* namespaces). Since the only happened on Linux so + // far, we hardcode the library name, load the symbols into the global + // namespace and leak the handle. + dlopen("libmono-2.0.so", RTLD_LAZY | RTLD_GLOBAL); +#endif + if (ext == 0) { pn_args->init_name = "Python.Runtime:Initialize()"; @@ -91,9 +100,9 @@ MonoMethod *getMethodFromClass(MonoClass *cls, char *name) return method; } -void main_thread_handler(gpointer user_data) +void main_thread_handler(PyNet_Args *user_data) { - PyNet_Args *pn_args = (PyNet_Args *)user_data; + PyNet_Args *pn_args = user_data; MonoMethod *init; MonoImage *pr_image; MonoClass *pythonengine; @@ -122,7 +131,7 @@ void main_thread_handler(gpointer user_data) strcpy(new_ld_library_path, py_libdir); strcat(new_ld_library_path, ":"); strcat(new_ld_library_path, ld_library_path); - setenv("LD_LIBRARY_PATH", py_libdir, 1); + setenv("LD_LIBRARY_PATH", new_ld_library_path, 1); } } @@ -241,7 +250,7 @@ void main_thread_handler(gpointer user_data) // Get string from a Mono exception char *PyNet_ExceptionToString(MonoObject *e) { - MonoMethodDesc *mdesc = mono_method_desc_new(":ToString()", FALSE); + MonoMethodDesc *mdesc = mono_method_desc_new(":ToString()", 0 /*FALSE*/); MonoMethod *mmethod = mono_method_desc_search_in_class(mdesc, mono_get_object_class()); mono_method_desc_free(mdesc); diff --git a/src/perf_tests/BaselineComparisonBenchmarkBase.cs b/src/perf_tests/BaselineComparisonBenchmarkBase.cs new file mode 100644 index 000000000..2388e3982 --- /dev/null +++ b/src/perf_tests/BaselineComparisonBenchmarkBase.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +using Python.Runtime; + +namespace Python.PerformanceTests +{ + public class BaselineComparisonBenchmarkBase + { + public BaselineComparisonBenchmarkBase() + { + Console.WriteLine($"CWD: {Environment.CurrentDirectory}"); + Console.WriteLine($"Using Python.Runtime from {typeof(PythonEngine).Assembly.Location} {typeof(PythonEngine).Assembly.GetName()}"); + + try { + PythonEngine.Initialize(); + Console.WriteLine("Python Initialized"); + if (PythonEngine.BeginAllowThreads() == IntPtr.Zero) + throw new PythonException(); + Console.WriteLine("Threading enabled"); + } + catch (Exception e) { + Console.WriteLine(e); + throw; + } + } + + static BaselineComparisonBenchmarkBase() + { + string pythonRuntimeDll = Environment.GetEnvironmentVariable(BaselineComparisonConfig.EnvironmentVariableName); + if (string.IsNullOrEmpty(pythonRuntimeDll)) + { + throw new ArgumentException( + "Required environment variable is missing", + BaselineComparisonConfig.EnvironmentVariableName); + } + + Console.WriteLine("Preloading " + pythonRuntimeDll); + Assembly.LoadFrom(pythonRuntimeDll); + foreach (var assembly in AppDomain.CurrentDomain.GetAssemblies()) { + if (assembly.FullName.StartsWith("Python.Runtime")) + Console.WriteLine(assembly.Location); + foreach(var dependency in assembly.GetReferencedAssemblies()) + if (dependency.FullName.Contains("Python.Runtime")) { + Console.WriteLine($"{assembly} -> {dependency}"); + } + } + + AppDomain.CurrentDomain.AssemblyResolve += CurrentDomainOnAssemblyResolve; + } + + static Assembly CurrentDomainOnAssemblyResolve(object sender, ResolveEventArgs args) { + if (!args.Name.StartsWith("Python.Runtime")) + return null; + + var preloaded = AppDomain.CurrentDomain.GetAssemblies() + .FirstOrDefault(a => a.GetName().Name == "Python.Runtime"); + if (preloaded != null) return preloaded; + + string pythonRuntimeDll = Environment.GetEnvironmentVariable(BaselineComparisonConfig.EnvironmentVariableName); + if (string.IsNullOrEmpty(pythonRuntimeDll)) + return null; + + return Assembly.LoadFrom(pythonRuntimeDll); + } + } +} diff --git a/src/perf_tests/BaselineComparisonConfig.cs b/src/perf_tests/BaselineComparisonConfig.cs new file mode 100644 index 000000000..649bb56fd --- /dev/null +++ b/src/perf_tests/BaselineComparisonConfig.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Reflection; + +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Horology; + +namespace Python.PerformanceTests +{ + public class BaselineComparisonConfig : ManualConfig + { + public const string EnvironmentVariableName = "PythonRuntimeDLL"; + + public BaselineComparisonConfig() + { + this.Options |= ConfigOptions.DisableOptimizationsValidator; + + string deploymentRoot = BenchmarkTests.DeploymentRoot; + + var baseJob = Job.Default + .WithLaunchCount(1) + .WithWarmupCount(3) + .WithMaxIterationCount(100) + .WithIterationTime(TimeInterval.FromMilliseconds(100)); + this.Add(baseJob + .WithId("baseline") + .WithEnvironmentVariable(EnvironmentVariableName, + Path.Combine(deploymentRoot, "baseline", "Python.Runtime.dll")) + .WithBaseline(true)); + this.Add(baseJob + .WithId("new") + .WithEnvironmentVariable(EnvironmentVariableName, + Path.Combine(deploymentRoot, "new", "Python.Runtime.dll"))); + } + + static BaselineComparisonConfig() { + AppDomain.CurrentDomain.AssemblyResolve += CurrentDomainOnAssemblyResolve; + } + + static Assembly CurrentDomainOnAssemblyResolve(object sender, ResolveEventArgs args) { + Console.WriteLine(args.Name); + if (!args.Name.StartsWith("Python.Runtime")) + return null; + string pythonRuntimeDll = Environment.GetEnvironmentVariable(EnvironmentVariableName); + if (string.IsNullOrEmpty(pythonRuntimeDll)) + pythonRuntimeDll = Path.Combine(BenchmarkTests.DeploymentRoot, "baseline", "Python.Runtime.dll"); + return Assembly.LoadFrom(pythonRuntimeDll); + } + } +} diff --git a/src/perf_tests/BenchmarkTests.cs b/src/perf_tests/BenchmarkTests.cs new file mode 100644 index 000000000..6e0afca69 --- /dev/null +++ b/src/perf_tests/BenchmarkTests.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Reflection; + +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using NUnit.Framework; + +namespace Python.PerformanceTests +{ + public class BenchmarkTests + { + Summary summary; + + [OneTimeSetUp] + public void SetUp() + { + Environment.CurrentDirectory = Path.Combine(DeploymentRoot, "new"); + this.summary = BenchmarkRunner.Run(); + Assert.IsNotEmpty(this.summary.Reports); + Assert.IsTrue( + condition: this.summary.Reports.All(r => r.Success), + message: "BenchmarkDotNet failed to execute or collect results of performance tests. See logs above."); + } + + [Test] + public void ReadInt64Property() + { + double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports); + AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.57); + } + + [Test] + public void WriteInt64Property() + { + double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports); + AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.57); + } + + static double GetOptimisticPerfRatio( + IReadOnlyList reports, + [CallerMemberName] string methodName = null) + { + reports = reports.Where(r => r.BenchmarkCase.Descriptor.WorkloadMethod.Name == methodName).ToArray(); + if (reports.Count == 0) + throw new ArgumentException( + message: $"No reports found for {methodName}. " + + "You have to match test method name to benchmark method name or " + + "pass benchmark method name explicitly", + paramName: nameof(methodName)); + + var baseline = reports.Single(r => r.BenchmarkCase.Job.ResolvedId == "baseline").ResultStatistics; + var @new = reports.Single(r => r.BenchmarkCase.Job.ResolvedId != "baseline").ResultStatistics; + + double newTimeOptimistic = @new.Mean - (@new.StandardDeviation + baseline.StandardDeviation) * 0.5; + + return newTimeOptimistic / baseline.Mean; + } + + public static string DeploymentRoot => Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + + public static void AssertPerformanceIsBetterOrSame( + double actual, double target, + double wiggleRoom = 1.1, [CallerMemberName] string testName = null) { + double threshold = target * wiggleRoom; + Assert.LessOrEqual(actual, threshold, + $"{testName}: {actual:F3} > {threshold:F3} (target: {target:F3})" + + ": perf result is higher than the failure threshold."); + } + } +} diff --git a/src/perf_tests/Python.PerformanceTests.csproj b/src/perf_tests/Python.PerformanceTests.csproj new file mode 100644 index 000000000..25af89db0 --- /dev/null +++ b/src/perf_tests/Python.PerformanceTests.csproj @@ -0,0 +1,44 @@ + + + + net461 + DebugMono;DebugMonoPY3;ReleaseMono;ReleaseMonoPY3;DebugWin;DebugWinPY3;ReleaseWin;ReleaseWinPY3 + bin\ + + false + + x64;x86 + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + compile + + + + + + + + + + + + + + + + + + diff --git a/src/perf_tests/PythonCallingNetBenchmark.cs b/src/perf_tests/PythonCallingNetBenchmark.cs new file mode 100644 index 000000000..ef668a911 --- /dev/null +++ b/src/perf_tests/PythonCallingNetBenchmark.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Text; + +using BenchmarkDotNet.Attributes; +using Python.Runtime; + +namespace Python.PerformanceTests +{ + [Config(typeof(BaselineComparisonConfig))] + public class PythonCallingNetBenchmark: BaselineComparisonBenchmarkBase + { + [Benchmark] + public void ReadInt64Property() + { + using (Py.GIL()) + { + var locals = new PyDict(); + locals.SetItem("a", new NetObject().ToPython()); + PythonEngine.Exec($@" +s = 0 +for i in range(50000): + s += a.{nameof(NetObject.LongProperty)} +", locals: locals.Handle); + } + } + + [Benchmark] + public void WriteInt64Property() { + using (Py.GIL()) { + var locals = new PyDict(); + locals.SetItem("a", new NetObject().ToPython()); + PythonEngine.Exec($@" +s = 0 +for i in range(50000): + a.{nameof(NetObject.LongProperty)} += i +", locals: locals.Handle); + } + } + } + + class NetObject + { + public long LongProperty { get; set; } = 42; + } +} diff --git a/src/runtime/BorrowedReference.cs b/src/runtime/BorrowedReference.cs new file mode 100644 index 000000000..a3bf29056 --- /dev/null +++ b/src/runtime/BorrowedReference.cs @@ -0,0 +1,25 @@ +namespace Python.Runtime +{ + using System; + /// + /// Represents a reference to a Python object, that is being lent, and + /// can only be safely used until execution returns to the caller. + /// + readonly ref struct BorrowedReference + { + readonly IntPtr pointer; + public bool IsNull => this.pointer == IntPtr.Zero; + + /// Gets a raw pointer to the Python object + public IntPtr DangerousGetAddress() + => this.IsNull ? throw new NullReferenceException() : this.pointer; + + /// + /// Creates new instance of from raw pointer. Unsafe. + /// + public BorrowedReference(IntPtr pointer) + { + this.pointer = pointer; + } + } +} diff --git a/src/runtime/Codecs/DecoderGroup.cs b/src/runtime/Codecs/DecoderGroup.cs new file mode 100644 index 000000000..8a290d5d4 --- /dev/null +++ b/src/runtime/Codecs/DecoderGroup.cs @@ -0,0 +1,78 @@ +namespace Python.Runtime.Codecs +{ + using System; + using System.Collections; + using System.Collections.Generic; + using System.Linq; + + /// + /// Represents a group of s. Useful to group them by priority. + /// + [Obsolete(Util.UnstableApiMessage)] + public sealed class DecoderGroup: IPyObjectDecoder, IEnumerable + { + readonly List decoders = new List(); + + /// + /// Add specified decoder to the group + /// + public void Add(IPyObjectDecoder item) + { + if (item is null) throw new ArgumentNullException(nameof(item)); + + this.decoders.Add(item); + } + /// + /// Remove all decoders from the group + /// + public void Clear() => this.decoders.Clear(); + + /// + public bool CanDecode(PyObject objectType, Type targetType) + => this.decoders.Any(decoder => decoder.CanDecode(objectType, targetType)); + /// + public bool TryDecode(PyObject pyObj, out T value) + { + if (pyObj is null) throw new ArgumentNullException(nameof(pyObj)); + + var decoder = this.GetDecoder(pyObj.GetPythonType(), typeof(T)); + if (decoder is null) + { + value = default; + return false; + } + return decoder.TryDecode(pyObj, out value); + } + + /// + public IEnumerator GetEnumerator() => this.decoders.GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => this.decoders.GetEnumerator(); + } + + [Obsolete(Util.UnstableApiMessage)] + public static class DecoderGroupExtensions + { + /// + /// Gets a concrete instance of + /// (potentially selecting one from a collection), + /// that can decode from to , + /// or null if a matching decoder can not be found. + /// + [Obsolete(Util.UnstableApiMessage)] + public static IPyObjectDecoder GetDecoder( + this IPyObjectDecoder decoder, + PyObject objectType, Type targetType) + { + if (decoder is null) throw new ArgumentNullException(nameof(decoder)); + + if (decoder is IEnumerable composite) + { + return composite + .Select(nestedDecoder => nestedDecoder.GetDecoder(objectType, targetType)) + .FirstOrDefault(d => d != null); + } + + return decoder.CanDecode(objectType, targetType) ? decoder : null; + } + } +} diff --git a/src/runtime/Codecs/EncoderGroup.cs b/src/runtime/Codecs/EncoderGroup.cs new file mode 100644 index 000000000..a5708c0bb --- /dev/null +++ b/src/runtime/Codecs/EncoderGroup.cs @@ -0,0 +1,79 @@ +namespace Python.Runtime.Codecs +{ + using System; + using System.Collections; + using System.Collections.Generic; + using System.Linq; + + /// + /// Represents a group of s. Useful to group them by priority. + /// + [Obsolete(Util.UnstableApiMessage)] + public sealed class EncoderGroup: IPyObjectEncoder, IEnumerable + { + readonly List encoders = new List(); + + /// + /// Add specified encoder to the group + /// + public void Add(IPyObjectEncoder item) + { + if (item is null) throw new ArgumentNullException(nameof(item)); + this.encoders.Add(item); + } + /// + /// Remove all encoders from the group + /// + public void Clear() => this.encoders.Clear(); + + /// + public bool CanEncode(Type type) => this.encoders.Any(encoder => encoder.CanEncode(type)); + /// + public PyObject TryEncode(object value) + { + if (value is null) throw new ArgumentNullException(nameof(value)); + + foreach (var encoder in this.GetEncoders(value.GetType())) + { + var result = encoder.TryEncode(value); + if (result != null) + { + return result; + } + } + + return null; + } + + /// + public IEnumerator GetEnumerator() => this.encoders.GetEnumerator(); + IEnumerator IEnumerable.GetEnumerator() => this.encoders.GetEnumerator(); + } + + [Obsolete(Util.UnstableApiMessage)] + public static class EncoderGroupExtensions + { + /// + /// Gets specific instances of + /// (potentially selecting one from a collection), + /// that can encode the specified . + /// + [Obsolete(Util.UnstableApiMessage)] + public static IEnumerable GetEncoders(this IPyObjectEncoder decoder, Type type) + { + if (decoder is null) throw new ArgumentNullException(nameof(decoder)); + + if (decoder is IEnumerable composite) + { + foreach (var nestedEncoder in composite) + foreach (var match in nestedEncoder.GetEncoders(type)) + { + yield return match; + } + } else if (decoder.CanEncode(type)) + { + yield return decoder; + } + } + } +} diff --git a/src/runtime/Codecs/RawProxyEncoder.cs b/src/runtime/Codecs/RawProxyEncoder.cs new file mode 100644 index 000000000..a1b6c52b3 --- /dev/null +++ b/src/runtime/Codecs/RawProxyEncoder.cs @@ -0,0 +1,21 @@ +using System; + +namespace Python.Runtime.Codecs +{ + /// + /// A .NET object encoder, that returns raw proxies (e.g. no conversion to Python types). + /// You must inherit from this class and override . + /// + [Obsolete(Util.UnstableApiMessage)] + public class RawProxyEncoder: IPyObjectEncoder + { + public PyObject TryEncode(object value) + { + if (value is null) throw new ArgumentNullException(nameof(value)); + + return PyObject.FromManagedObject(value); + } + + public virtual bool CanEncode(Type type) => false; + } +} diff --git a/src/runtime/Codecs/TupleCodecs.cs b/src/runtime/Codecs/TupleCodecs.cs new file mode 100644 index 000000000..a9ae33fe0 --- /dev/null +++ b/src/runtime/Codecs/TupleCodecs.cs @@ -0,0 +1,135 @@ +namespace Python.Runtime.Codecs +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Reflection; + + [Obsolete(Util.UnstableApiMessage)] + public sealed class TupleCodec : IPyObjectEncoder, IPyObjectDecoder + { + TupleCodec() { } + public static TupleCodec Instance { get; } = new TupleCodec(); + + public bool CanEncode(Type type) + { + if (type == typeof(object) || type == typeof(TTuple)) return true; + return type.Namespace == typeof(TTuple).Namespace + // generic versions of tuples are named Tuple`TYPE_ARG_COUNT + && type.Name.StartsWith(typeof(TTuple).Name + '`'); + } + + public PyObject TryEncode(object value) + { + if (value == null) return null; + + var tupleType = value.GetType(); + if (tupleType == typeof(object)) return null; + if (!this.CanEncode(tupleType)) return null; + if (tupleType == typeof(TTuple)) return new PyTuple(); + + long fieldCount = tupleType.GetGenericArguments().Length; + var tuple = Runtime.PyTuple_New(fieldCount); + Exceptions.ErrorCheck(tuple); + int fieldIndex = 0; + foreach (FieldInfo field in tupleType.GetFields()) + { + var item = field.GetValue(value); + IntPtr pyItem = Converter.ToPython(item); + Runtime.PyTuple_SetItem(tuple, fieldIndex, pyItem); + fieldIndex++; + } + return new PyTuple(tuple); + } + + public bool CanDecode(PyObject objectType, Type targetType) + => objectType.Handle == Runtime.PyTupleType && this.CanEncode(targetType); + + public bool TryDecode(PyObject pyObj, out T value) + { + if (pyObj == null) throw new ArgumentNullException(nameof(pyObj)); + + value = default; + + if (!Runtime.PyTuple_Check(pyObj.Handle)) return false; + + if (typeof(T) == typeof(object)) + { + bool converted = Decode(pyObj, out object result); + if (converted) + { + value = (T)result; + return true; + } + + return false; + } + + var itemTypes = typeof(T).GetGenericArguments(); + long itemCount = Runtime.PyTuple_Size(pyObj.Handle); + if (itemTypes.Length != itemCount) return false; + + if (itemCount == 0) + { + value = (T)EmptyTuple; + return true; + } + + var elements = new object[itemCount]; + for (int itemIndex = 0; itemIndex < itemTypes.Length; itemIndex++) + { + IntPtr pyItem = Runtime.PyTuple_GetItem(pyObj.Handle, itemIndex); + if (!Converter.ToManaged(pyItem, itemTypes[itemIndex], out elements[itemIndex], setError: false)) + { + Exceptions.Clear(); + return false; + } + } + var factory = tupleCreate[itemCount].MakeGenericMethod(itemTypes); + value = (T)factory.Invoke(null, elements); + return true; + } + + static bool Decode(PyObject tuple, out object value) + { + long itemCount = Runtime.PyTuple_Size(tuple.Handle); + if (itemCount == 0) + { + value = EmptyTuple; + return true; + } + var elements = new object[itemCount]; + var itemTypes = new Type[itemCount]; + value = null; + for (int itemIndex = 0; itemIndex < elements.Length; itemIndex++) + { + var pyItem = Runtime.PyTuple_GetItem(tuple.Handle, itemIndex); + if (!Converter.ToManaged(pyItem, typeof(object), out elements[itemIndex], setError: false)) + { + Exceptions.Clear(); + return false; + } + + itemTypes[itemIndex] = elements[itemIndex]?.GetType() ?? typeof(object); + } + + var factory = tupleCreate[itemCount].MakeGenericMethod(itemTypes); + value = factory.Invoke(null, elements); + return true; + } + + static readonly MethodInfo[] tupleCreate = + typeof(TTuple).GetMethods(BindingFlags.Public | BindingFlags.Static) + .Where(m => m.Name == nameof(Tuple.Create)) + .OrderBy(m => m.GetParameters().Length) + .ToArray(); + + static readonly object EmptyTuple = tupleCreate[0].Invoke(null, parameters: new object[0]); + + public static void Register() + { + PyObjectConversions.RegisterEncoder(Instance); + PyObjectConversions.RegisterDecoder(Instance); + } + } +} diff --git a/src/runtime/NewReference.cs b/src/runtime/NewReference.cs new file mode 100644 index 000000000..6e66232d0 --- /dev/null +++ b/src/runtime/NewReference.cs @@ -0,0 +1,70 @@ +namespace Python.Runtime +{ + using System; + using System.Diagnostics.Contracts; + + /// + /// Represents a reference to a Python object, that is tracked by Python's reference counting. + /// + [NonCopyable] + ref struct NewReference + { + IntPtr pointer; + + [Pure] + public static implicit operator BorrowedReference(in NewReference reference) + => new BorrowedReference(reference.pointer); + + /// + /// Returns wrapper around this reference, which now owns + /// the pointer. Sets the original reference to null, as it no longer owns it. + /// + public PyObject MoveToPyObject() + { + if (this.IsNull()) throw new NullReferenceException(); + + var result = new PyObject(this.pointer); + this.pointer = IntPtr.Zero; + return result; + } + /// + /// Removes this reference to a Python object, and sets it to null. + /// + public void Dispose() + { + if (!this.IsNull()) + Runtime.XDecref(this.pointer); + this.pointer = IntPtr.Zero; + } + + /// + /// Creates from a raw pointer + /// + [Pure] + public static NewReference DangerousFromPointer(IntPtr pointer) + => new NewReference {pointer = pointer}; + + [Pure] + internal static IntPtr DangerousGetAddress(in NewReference reference) + => IsNull(reference) ? throw new NullReferenceException() : reference.pointer; + [Pure] + internal static bool IsNull(in NewReference reference) + => reference.pointer == IntPtr.Zero; + } + + /// + /// These members can not be directly in type, + /// because this is always passed by value, which we need to avoid. + /// (note this in NewReference vs the usual this NewReference) + /// + static class NewReferenceExtensions + { + /// Gets a raw pointer to the Python object + [Pure] + public static IntPtr DangerousGetAddress(this in NewReference reference) + => NewReference.DangerousGetAddress(reference); + [Pure] + public static bool IsNull(this in NewReference reference) + => NewReference.IsNull(reference); + } +} diff --git a/src/runtime/NonCopyableAttribute.cs b/src/runtime/NonCopyableAttribute.cs new file mode 100644 index 000000000..63d36ab42 --- /dev/null +++ b/src/runtime/NonCopyableAttribute.cs @@ -0,0 +1,6 @@ +namespace Python.Runtime +{ + using System; + [AttributeUsage(AttributeTargets.Struct)] + class NonCopyableAttribute : Attribute { } +} diff --git a/src/runtime/Properties/AssemblyInfo.cs b/src/runtime/Properties/AssemblyInfo.cs index a5d33c7ab..cab9df30b 100644 --- a/src/runtime/Properties/AssemblyInfo.cs +++ b/src/runtime/Properties/AssemblyInfo.cs @@ -4,7 +4,7 @@ // General Information about an assembly is controlled through the following // set of attributes. Change these attribute values to modify the information // associated with an assembly. -[assembly: AssemblyTitle("Python for .NET")] +[assembly: AssemblyTitle("Python.NET")] [assembly: AssemblyDescription("")] [assembly: AssemblyDefaultAlias("Python.Runtime.dll")] diff --git a/src/runtime/PyExportAttribute.cs b/src/runtime/PyExportAttribute.cs new file mode 100644 index 000000000..52a8be15d --- /dev/null +++ b/src/runtime/PyExportAttribute.cs @@ -0,0 +1,16 @@ +namespace Python.Runtime { + using System; + + /// + /// Controls visibility to Python for public .NET type or an entire assembly + /// + [AttributeUsage(AttributeTargets.Class | AttributeTargets.Delegate | AttributeTargets.Enum + | AttributeTargets.Interface | AttributeTargets.Struct | AttributeTargets.Assembly, + AllowMultiple = false, + Inherited = false)] + public class PyExportAttribute : Attribute + { + internal readonly bool Export; + public PyExportAttribute(bool export) { this.Export = export; } + } +} diff --git a/src/runtime/Python.Runtime.15.csproj b/src/runtime/Python.Runtime.15.csproj index 29177b78c..d753a5dff 100644 --- a/src/runtime/Python.Runtime.15.csproj +++ b/src/runtime/Python.Runtime.15.csproj @@ -7,14 +7,21 @@ net45 Python.Runtime Python.Runtime - Python.Runtime - 2.4.0 - false - false - false - false - false - false + pythonnet + 2.5.0 + true + false + Python.NET + Copyright (c) 2006-2020 the contributors of the Python.NET project + Python and CLR (.NET and Mono) cross-platform language interop + pythonnet + https://github.com/pythonnet/pythonnet/blob/master/LICENSE + https://github.com/pythonnet/pythonnet + git + + python interop dynamic dlr Mono pinvoke + https://raw.githubusercontent.com/pythonnet/pythonnet/master/src/console/python-clear.ico + https://pythonnet.github.io/ bin\ false $(OutputPath)\$(AssemblyName).xml @@ -23,7 +30,7 @@ ..\..\ $(SolutionDir)\bin\ $(PythonBuildDir)\$(TargetFramework)\ - 6 + 7.3 True ..\pythonnet.snk $(PYTHONNET_DEFINE_CONSTANTS) @@ -35,7 +42,7 @@ $(PYTHONNET_PY2_VERSION) PYTHON27 $(PYTHONNET_PY3_VERSION) - PYTHON37 + PYTHON38 $(PYTHONNET_WIN_DEFINE_CONSTANTS) UCS2 $(PYTHONNET_MONO_DEFINE_CONSTANTS) @@ -122,6 +129,13 @@ + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + diff --git a/src/runtime/Python.Runtime.csproj b/src/runtime/Python.Runtime.csproj index 19f776c77..75f5e2fab 100644 --- a/src/runtime/Python.Runtime.csproj +++ b/src/runtime/Python.Runtime.csproj @@ -1,4 +1,4 @@ - + Debug @@ -15,18 +15,18 @@ ..\..\ $(SolutionDir)\bin\ Properties - 6 + 7.3 true false ..\pythonnet.snk - PYTHON2;PYTHON27;UCS4 @@ -34,7 +34,7 @@ pdbonly - PYTHON3;PYTHON37;UCS4 + PYTHON3;PYTHON38;UCS4 true pdbonly @@ -46,7 +46,7 @@ true - PYTHON3;PYTHON37;UCS4;TRACE;DEBUG + PYTHON3;PYTHON38;UCS4;TRACE;DEBUG false full @@ -56,7 +56,7 @@ pdbonly - PYTHON3;PYTHON37;UCS2 + PYTHON3;PYTHON38;UCS2 true pdbonly @@ -68,7 +68,7 @@ true - PYTHON3;PYTHON37;UCS2;TRACE;DEBUG + PYTHON3;PYTHON38;UCS2;TRACE;DEBUG false full @@ -76,6 +76,11 @@ + + + + + @@ -83,6 +88,7 @@ + @@ -119,10 +125,13 @@ + + + @@ -136,10 +145,15 @@ + + + + + @@ -149,7 +163,8 @@ - + + @@ -168,4 +183,4 @@ - + \ No newline at end of file diff --git a/src/runtime/ReferenceExtensions.cs b/src/runtime/ReferenceExtensions.cs new file mode 100644 index 000000000..8fa2731b7 --- /dev/null +++ b/src/runtime/ReferenceExtensions.cs @@ -0,0 +1,20 @@ +namespace Python.Runtime +{ + using System.Diagnostics.Contracts; + + static class ReferenceExtensions + { + /// + /// Checks if the reference points to Python object None. + /// + [Pure] + public static bool IsNone(this in NewReference reference) + => reference.DangerousGetAddress() == Runtime.PyNone; + /// + /// Checks if the reference points to Python object None. + /// + [Pure] + public static bool IsNone(this BorrowedReference reference) + => reference.DangerousGetAddress() == Runtime.PyNone; + } +} diff --git a/src/runtime/Util.cs b/src/runtime/Util.cs index dc5f78608..eb21cddbb 100644 --- a/src/runtime/Util.cs +++ b/src/runtime/Util.cs @@ -3,8 +3,11 @@ namespace Python.Runtime { - internal class Util + internal static class Util { + internal const string UnstableApiMessage = + "This API is unstable, and might be changed or removed in the next minor release"; + internal static Int64 ReadCLong(IntPtr tp, int offset) { // On Windows, a C long is always 32 bits. @@ -29,5 +32,12 @@ internal static void WriteCLong(IntPtr type, int offset, Int64 flags) Marshal.WriteInt64(type, offset, flags); } } + + /// + /// Null-coalesce: if parameter is not + /// , return it. Otherwise return . + /// + internal static IntPtr Coalesce(this IntPtr primary, IntPtr fallback) + => primary == IntPtr.Zero ? fallback : primary; } -} \ No newline at end of file +} diff --git a/src/runtime/arrayobject.cs b/src/runtime/arrayobject.cs index c37295704..1ef318473 100644 --- a/src/runtime/arrayobject.cs +++ b/src/runtime/arrayobject.cs @@ -244,16 +244,5 @@ public static int sq_contains(IntPtr ob, IntPtr v) return 0; } - - - /// - /// Implements __len__ for array types. - /// - public static int mp_length(IntPtr ob) - { - var self = (CLRObject)GetManagedObject(ob); - var items = self.inst as Array; - return items.Length; - } } } diff --git a/src/runtime/assemblymanager.cs b/src/runtime/assemblymanager.cs index 3085bb639..46909a370 100644 --- a/src/runtime/assemblymanager.cs +++ b/src/runtime/assemblymanager.cs @@ -145,7 +145,7 @@ internal static void UpdatePath() probed.Clear(); for (var i = 0; i < count; i++) { - IntPtr item = Runtime.PyList_GetItem(list, i); + BorrowedReference item = Runtime.PyList_GetItem(list, i); string path = Runtime.GetManagedString(item); if (path != null) { @@ -346,6 +346,10 @@ public static bool LoadImplicit(string name, bool warn = true) /// internal static void ScanAssembly(Assembly assembly) { + if (assembly.GetCustomAttribute()?.Export == false) + { + return; + } // A couple of things we want to do here: first, we want to // gather a list of all of the namespaces contributed to by // the assembly. @@ -452,12 +456,13 @@ public static List GetNames(string nsname) /// looking in the currently loaded assemblies for the named /// type. Returns null if the named type cannot be found. /// + [Obsolete("Use LookupTypes and handle name conflicts")] public static Type LookupType(string qname) { foreach (Assembly assembly in assemblies) { Type type = assembly.GetType(qname); - if (type != null) + if (type != null && IsExported(type)) { return type; } @@ -465,25 +470,33 @@ public static Type LookupType(string qname) return null; } + /// + /// Returns the objects for the given qualified name, + /// looking in the currently loaded assemblies for the named + /// type. + /// + public static IEnumerable LookupTypes(string qualifiedName) + => assemblies.Select(assembly => assembly.GetType(qualifiedName)).Where(type => type != null && IsExported(type)); + internal static Type[] GetTypes(Assembly a) { if (a.IsDynamic) { try { - return a.GetTypes(); + return a.GetTypes().Where(IsExported).ToArray(); } catch (ReflectionTypeLoadException exc) { // Return all types that were successfully loaded - return exc.Types.Where(x => x != null).ToArray(); + return exc.Types.Where(x => x != null && IsExported(x)).ToArray(); } } else { try { - return a.GetExportedTypes(); + return a.GetExportedTypes().Where(IsExported).ToArray(); } catch (FileNotFoundException) { @@ -491,5 +504,7 @@ internal static Type[] GetTypes(Assembly a) } } } + + static bool IsExported(Type type) => type.GetCustomAttribute()?.Export != false; } -} \ No newline at end of file +} diff --git a/src/runtime/classbase.cs b/src/runtime/classbase.cs index 5846fa82a..144bac9d3 100644 --- a/src/runtime/classbase.cs +++ b/src/runtime/classbase.cs @@ -246,6 +246,44 @@ public static IntPtr tp_str(IntPtr ob) } } + public static IntPtr tp_repr(IntPtr ob) + { + var co = GetManagedObject(ob) as CLRObject; + if (co == null) + { + return Exceptions.RaiseTypeError("invalid object"); + } + try + { + //if __repr__ is defined, use it + var instType = co.inst.GetType(); + System.Reflection.MethodInfo methodInfo = instType.GetMethod("__repr__"); + if (methodInfo != null && methodInfo.IsPublic) + { + var reprString = methodInfo.Invoke(co.inst, null) as string; + return Runtime.PyString_FromString(reprString); + } + + //otherwise use the standard object.__repr__(inst) + IntPtr args = Runtime.PyTuple_New(1); + Runtime.PyTuple_SetItem(args, 0, ob); + IntPtr reprFunc = Runtime.PyObject_GetAttrString(Runtime.PyBaseObjectType, "__repr__"); + var output = Runtime.PyObject_Call(reprFunc, args, IntPtr.Zero); + Runtime.XDecref(args); + Runtime.XDecref(reprFunc); + return output; + } + catch (Exception e) + { + if (e.InnerException != null) + { + e = e.InnerException; + } + Exceptions.SetError(e); + return IntPtr.Zero; + } + } + /// /// Standard dealloc implementation for instances of reflected types. @@ -253,7 +291,7 @@ public static IntPtr tp_str(IntPtr ob) public static void tp_dealloc(IntPtr ob) { ManagedType self = GetManagedObject(ob); - IntPtr dict = Marshal.ReadIntPtr(ob, ObjectOffset.DictOffset(ob)); + IntPtr dict = Marshal.ReadIntPtr(ob, ObjectOffset.TypeDictOffset(self.tpHandle)); if (dict != IntPtr.Zero) { Runtime.XDecref(dict); diff --git a/src/runtime/classderived.cs b/src/runtime/classderived.cs index ec3734ea5..af16b1359 100644 --- a/src/runtime/classderived.cs +++ b/src/runtime/classderived.cs @@ -877,7 +877,7 @@ public static void Finalize(IPythonDerivedType obj) // the C# object is being destroyed which must mean there are no more // references to the Python object as well so now we can dealloc the // python object. - IntPtr dict = Marshal.ReadIntPtr(self.pyHandle, ObjectOffset.DictOffset(self.pyHandle)); + IntPtr dict = Marshal.ReadIntPtr(self.pyHandle, ObjectOffset.TypeDictOffset(self.tpHandle)); if (dict != IntPtr.Zero) { Runtime.XDecref(dict); diff --git a/src/runtime/clrobject.cs b/src/runtime/clrobject.cs index 502677655..5c7ad7891 100644 --- a/src/runtime/clrobject.cs +++ b/src/runtime/clrobject.cs @@ -14,11 +14,11 @@ internal CLRObject(object ob, IntPtr tp) long flags = Util.ReadCLong(tp, TypeOffset.tp_flags); if ((flags & TypeFlags.Subclass) != 0) { - IntPtr dict = Marshal.ReadIntPtr(py, ObjectOffset.DictOffset(tp)); + IntPtr dict = Marshal.ReadIntPtr(py, ObjectOffset.TypeDictOffset(tp)); if (dict == IntPtr.Zero) { dict = Runtime.PyDict_New(); - Marshal.WriteIntPtr(py, ObjectOffset.DictOffset(tp), dict); + Marshal.WriteIntPtr(py, ObjectOffset.TypeDictOffset(tp), dict); } } @@ -35,13 +35,13 @@ internal CLRObject(object ob, IntPtr tp) } - internal static CLRObject GetInstance(object ob, IntPtr pyType) + static CLRObject GetInstance(object ob, IntPtr pyType) { return new CLRObject(ob, pyType); } - internal static CLRObject GetInstance(object ob) + static CLRObject GetInstance(object ob) { ClassBase cc = ClassManager.GetClass(ob.GetType()); return GetInstance(ob, cc.tpHandle); diff --git a/src/runtime/constructorbinder.cs b/src/runtime/constructorbinder.cs index 1fc541920..0f9806c0e 100644 --- a/src/runtime/constructorbinder.cs +++ b/src/runtime/constructorbinder.cs @@ -1,5 +1,6 @@ using System; using System.Reflection; +using System.Text; namespace Python.Runtime { @@ -93,7 +94,15 @@ internal object InvokeRaw(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info) if (binding == null) { - Exceptions.SetError(Exceptions.TypeError, "no constructor matches given arguments"); + var errorMessage = new StringBuilder("No constructor matches given arguments"); + if (info != null && info.IsConstructor && info.DeclaringType != null) + { + errorMessage.Append(" for ").Append(info.DeclaringType.Name); + } + + errorMessage.Append(": "); + AppendArgumentTypes(to: errorMessage, args); + Exceptions.SetError(Exceptions.TypeError, errorMessage.ToString()); return null; } } diff --git a/src/runtime/converter.cs b/src/runtime/converter.cs index 11c67bf82..3add8aba0 100644 --- a/src/runtime/converter.cs +++ b/src/runtime/converter.cs @@ -135,8 +135,17 @@ internal static IntPtr ToPython(object value, Type type) return result; } - if (value is IList && !(value is INotifyPropertyChanged) && value.GetType().IsGenericType) - { + if (Type.GetTypeCode(type) == TypeCode.Object && value.GetType() != typeof(object)) { + var encoded = PyObjectConversions.TryEncode(value, type); + if (encoded != null) { + result = encoded.Handle; + Runtime.XIncref(result); + return result; + } + } + + if (value is IList && !(value is INotifyPropertyChanged) && value.GetType().IsGenericType) + { using (var resultlist = new PyList()) { foreach (object o in (IEnumerable)value) @@ -377,17 +386,21 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return ToPrimitive(value, doubleType, out result, setError); } - if (Runtime.PySequence_Check(value)) + // give custom codecs a chance to take over conversion of sequences + IntPtr pyType = Runtime.PyObject_TYPE(value); + if (PyObjectConversions.TryDecode(value, pyType, obType, out result)) { - return ToArray(value, typeof(object[]), out result, setError); + return true; } - if (setError) + if (Runtime.PySequence_Check(value)) { - Exceptions.SetError(Exceptions.TypeError, "value cannot be converted to Object"); + return ToArray(value, typeof(object[]), out result, setError); } - return false; + Runtime.XIncref(value); // PyObject() assumes ownership + result = new PyObject(value); + return true; } // Conversion to 'Type' is done using the same mappings as above for objects. @@ -437,9 +450,21 @@ internal static bool ToManagedValue(IntPtr value, Type obType, return false; } + TypeCode typeCode = Type.GetTypeCode(obType); + if (typeCode == TypeCode.Object) + { + IntPtr pyType = Runtime.PyObject_TYPE(value); + if (PyObjectConversions.TryDecode(value, pyType, obType, out result)) + { + return true; + } + } + return ToPrimitive(value, obType, out result, setError); } + internal delegate bool TryConvertFromPythonDelegate(IntPtr pyObj, out object result); + /// /// Convert a Python value to an instance of a primitive managed type. /// @@ -728,7 +753,20 @@ private static bool ToPrimitive(IntPtr value, Type obType, out object result, bo } goto type_error; } - uint ui = (uint)Runtime.PyLong_AsUnsignedLong(op); + + uint ui; + try + { + ui = Convert.ToUInt32(Runtime.PyLong_AsUnsignedLong(op)); + } catch (OverflowException) + { + // Probably wasn't an overflow in python but was in C# (e.g. if cpython + // longs are 64 bit then 0xFFFFFFFF + 1 will not overflow in + // PyLong_AsUnsignedLong) + Runtime.XDecref(op); + goto overflow; + } + if (Exceptions.ErrorOccurred()) { @@ -837,17 +875,20 @@ private static void SetConversionError(IntPtr value, Type target) /// /// Convert a Python value to a correctly typed managed array instance. - /// The Python value must support the Python sequence protocol and the + /// The Python value must support the Python iterator protocol or and the /// items in the sequence must be convertible to the target array type. /// private static bool ToArray(IntPtr value, Type obType, out object result, bool setError) { Type elementType = obType.GetElementType(); - var size = Runtime.PySequence_Size(value); result = null; - if (size < 0) - { + bool IsSeqObj = Runtime.PySequence_Check(value); + var len = IsSeqObj ? Runtime.PySequence_Size(value) : -1; + + IntPtr IterObject = Runtime.PyObject_GetIter(value); + + if(IterObject==IntPtr.Zero) { if (setError) { SetConversionError(value, obType); @@ -855,21 +896,17 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s return false; } - Array items = Array.CreateInstance(elementType, size); + Array items; + + var listType = typeof(List<>); + var constructedListType = listType.MakeGenericType(elementType); + IList list = IsSeqObj ? (IList) Activator.CreateInstance(constructedListType, new Object[] {(int) len}) : + (IList) Activator.CreateInstance(constructedListType); + IntPtr item; - // XXX - is there a better way to unwrap this if it is a real array? - for (var i = 0; i < size; i++) + while ((item = Runtime.PyIter_Next(IterObject)) != IntPtr.Zero) { object obj = null; - IntPtr item = Runtime.PySequence_GetItem(value, i); - if (item == IntPtr.Zero) - { - if (setError) - { - SetConversionError(value, obType); - return false; - } - } if (!Converter.ToManaged(item, elementType, out obj, true)) { @@ -877,10 +914,14 @@ private static bool ToArray(IntPtr value, Type obType, out object result, bool s return false; } - items.SetValue(obj, i); + list.Add(obj); Runtime.XDecref(item); } + Runtime.XDecref(IterObject); + items = Array.CreateInstance(elementType, list.Count); + list.CopyTo(items, 0); + result = items; return true; } diff --git a/src/runtime/converterextensions.cs b/src/runtime/converterextensions.cs new file mode 100644 index 000000000..667fc6f00 --- /dev/null +++ b/src/runtime/converterextensions.cs @@ -0,0 +1,192 @@ +namespace Python.Runtime +{ + using System; + using System.Collections.Concurrent; + using System.Collections.Generic; + using System.Linq; + using System.Reflection; + using Python.Runtime.Codecs; + + /// + /// Defines conversion to CLR types (unmarshalling) + /// + [Obsolete(Util.UnstableApiMessage)] + public interface IPyObjectDecoder + { + /// + /// Checks if this decoder can decode from to + /// + bool CanDecode(PyObject objectType, Type targetType); + /// + /// Attempts do decode into a variable of specified type + /// + /// CLR type to decode into + /// Object to decode + /// The variable, that will receive decoding result + /// + bool TryDecode(PyObject pyObj, out T value); + } + + /// + /// Defines conversion from CLR objects into Python objects (e.g. ) (marshalling) + /// + [Obsolete(Util.UnstableApiMessage)] + public interface IPyObjectEncoder + { + /// + /// Checks if encoder can encode CLR objects of specified type + /// + bool CanEncode(Type type); + /// + /// Attempts to encode CLR object into Python object + /// + PyObject TryEncode(object value); + } + + /// + /// This class allows to register additional marshalling codecs. + /// Python.NET will pick suitable encoder/decoder registered first + /// + [Obsolete(Util.UnstableApiMessage)] + public static class PyObjectConversions + { + static readonly DecoderGroup decoders = new DecoderGroup(); + static readonly EncoderGroup encoders = new EncoderGroup(); + + /// + /// Registers specified encoder (marshaller) + /// Python.NET will pick suitable encoder/decoder registered first + /// + public static void RegisterEncoder(IPyObjectEncoder encoder) + { + if (encoder == null) throw new ArgumentNullException(nameof(encoder)); + + lock (encoders) + { + encoders.Add(encoder); + } + } + + /// + /// Registers specified decoder (unmarshaller) + /// Python.NET will pick suitable encoder/decoder registered first + /// + public static void RegisterDecoder(IPyObjectDecoder decoder) + { + if (decoder == null) throw new ArgumentNullException(nameof(decoder)); + + lock (decoders) + { + decoders.Add(decoder); + } + } + + #region Encoding + internal static PyObject TryEncode(object obj, Type type) + { + if (obj == null) throw new ArgumentNullException(nameof(obj)); + if (type == null) throw new ArgumentNullException(nameof(type)); + + foreach (var encoder in clrToPython.GetOrAdd(type, GetEncoders)) + { + var result = encoder.TryEncode(obj); + if (result != null) return result; + } + + return null; + } + + static readonly ConcurrentDictionary + clrToPython = new ConcurrentDictionary(); + static IPyObjectEncoder[] GetEncoders(Type type) + { + lock (encoders) + { + return encoders.GetEncoders(type).ToArray(); + } + } + #endregion + + #region Decoding + static readonly ConcurrentDictionary + pythonToClr = new ConcurrentDictionary(); + internal static bool TryDecode(IntPtr pyHandle, IntPtr pyType, Type targetType, out object result) + { + if (pyHandle == IntPtr.Zero) throw new ArgumentNullException(nameof(pyHandle)); + if (pyType == IntPtr.Zero) throw new ArgumentNullException(nameof(pyType)); + if (targetType == null) throw new ArgumentNullException(nameof(targetType)); + + var decoder = pythonToClr.GetOrAdd(new TypePair(pyType, targetType), pair => GetDecoder(pair.PyType, pair.ClrType)); + result = null; + if (decoder == null) return false; + return decoder.Invoke(pyHandle, out result); + } + + static Converter.TryConvertFromPythonDelegate GetDecoder(IntPtr sourceType, Type targetType) + { + IPyObjectDecoder decoder; + using (var pyType = new PyObject(Runtime.SelfIncRef(sourceType))) + { + lock (decoders) + { + decoder = decoders.GetDecoder(pyType, targetType); + if (decoder == null) return null; + } + } + + var decode = genericDecode.MakeGenericMethod(targetType); + + bool TryDecode(IntPtr pyHandle, out object result) + { + var pyObj = new PyObject(Runtime.SelfIncRef(pyHandle)); + var @params = new object[] { pyObj, null }; + bool success = (bool)decode.Invoke(decoder, @params); + if (!success) + { + pyObj.Dispose(); + } + + result = @params[1]; + return success; + } + + return TryDecode; + } + + static readonly MethodInfo genericDecode = typeof(IPyObjectDecoder).GetMethod(nameof(IPyObjectDecoder.TryDecode)); + + #endregion + + internal static void Reset() + { + lock (encoders) + lock (decoders) + { + clrToPython.Clear(); + pythonToClr.Clear(); + encoders.Clear(); + decoders.Clear(); + } + } + + struct TypePair : IEquatable + { + internal readonly IntPtr PyType; + internal readonly Type ClrType; + + public TypePair(IntPtr pyType, Type clrType) + { + this.PyType = pyType; + this.ClrType = clrType; + } + + public override int GetHashCode() + => this.ClrType.GetHashCode() ^ this.PyType.GetHashCode(); + + public bool Equals(TypePair other) + => this.PyType == other.PyType && this.ClrType == other.ClrType; + + public override bool Equals(object obj) => obj is TypePair other && this.Equals(other); + } + } +} diff --git a/src/runtime/exceptions.cs b/src/runtime/exceptions.cs index 8bed0abfd..31c367eb2 100644 --- a/src/runtime/exceptions.cs +++ b/src/runtime/exceptions.cs @@ -36,6 +36,29 @@ internal static Exception ToException(IntPtr ob) return e; } + /// + /// Exception __repr__ implementation + /// + public new static IntPtr tp_repr(IntPtr ob) + { + Exception e = ToException(ob); + if (e == null) + { + return Exceptions.RaiseTypeError("invalid object"); + } + string name = e.GetType().Name; + string message; + if (e.Message != String.Empty) + { + message = String.Format("{0}('{1}')", name, e.Message); + } + else + { + message = String.Format("{0}()", name); + } + return Runtime.PyUnicode_FromString(message); + } + /// /// Exception __str__ implementation /// diff --git a/src/runtime/extensiontype.cs b/src/runtime/extensiontype.cs index 693a46f42..6585180c1 100644 --- a/src/runtime/extensiontype.cs +++ b/src/runtime/extensiontype.cs @@ -38,6 +38,7 @@ public ExtensionType() Runtime.PyObject_GC_UnTrack(py); + // Steals a ref to tpHandle. tpHandle = tp; pyHandle = py; gcHandle = gc; @@ -50,7 +51,7 @@ public ExtensionType() public static void FinalizeObject(ManagedType self) { Runtime.PyObject_GC_Del(self.pyHandle); - Runtime.XDecref(self.tpHandle); + // Not necessary for decref of `tpHandle`. self.gcHandle.Free(); } diff --git a/src/runtime/finalizer.cs b/src/runtime/finalizer.cs index dd5c0b4dd..ba562cc26 100644 --- a/src/runtime/finalizer.cs +++ b/src/runtime/finalizer.cs @@ -28,9 +28,7 @@ public class ErrorArgs : EventArgs public bool Enable { get; set; } private ConcurrentQueue _objQueue = new ConcurrentQueue(); - private bool _pending = false; - private readonly object _collectingLock = new object(); - private Task _finalizerTask; + private int _throttled; #region FINALIZER_CHECK @@ -75,19 +73,16 @@ private Finalizer() Threshold = 200; } - public void Collect(bool forceDispose = true) + [Obsolete("forceDispose parameter is unused. All objects are disposed regardless.")] + public void Collect(bool forceDispose) => this.DisposeAll(); + public void Collect() => this.DisposeAll(); + + internal void ThrottledCollect() { - if (Instance._finalizerTask != null - && !Instance._finalizerTask.IsCompleted) - { - var ts = PythonEngine.BeginAllowThreads(); - Instance._finalizerTask.Wait(); - PythonEngine.EndAllowThreads(ts); - } - else if (forceDispose) - { - Instance.DisposeAll(); - } + _throttled = unchecked(this._throttled + 1); + if (!Enable || _throttled < Threshold) return; + _throttled = 0; + this.Collect(); } public List GetCollectedObjects() @@ -101,62 +96,18 @@ internal void AddFinalizedObject(IPyDisposable obj) { return; } - if (Runtime.Py_IsInitialized() == 0) - { - // XXX: Memory will leak if a PyObject finalized after Python shutdown, - // for avoiding that case, user should call GC.Collect manual before shutdown. - return; - } + #if FINALIZER_CHECK lock (_queueLock) #endif { - _objQueue.Enqueue(obj); - } - GC.ReRegisterForFinalize(obj); - if (!_pending && _objQueue.Count >= Threshold) - { - AddPendingCollect(); + this._objQueue.Enqueue(obj); } } internal static void Shutdown() { - if (Runtime.Py_IsInitialized() == 0) - { - Instance._objQueue = new ConcurrentQueue(); - return; - } - Instance.Collect(forceDispose: true); - } - - private void AddPendingCollect() - { - if(Monitor.TryEnter(_collectingLock)) - { - try - { - if (!_pending) - { - _pending = true; - // should already be complete but just in case - _finalizerTask?.Wait(); - - _finalizerTask = Task.Factory.StartNew(() => - { - using (Py.GIL()) - { - Instance.DisposeAll(); - _pending = false; - } - }); - } - } - finally - { - Monitor.Exit(_collectingLock); - } - } + Instance.DisposeAll(); } private void DisposeAll() @@ -178,12 +129,18 @@ private void DisposeAll() try { obj.Dispose(); - Runtime.CheckExceptionOccurred(); } catch (Exception e) { - // We should not bother the main thread - ErrorHandler?.Invoke(this, new ErrorArgs() + var handler = ErrorHandler; + if (handler is null) + { + throw new FinalizationException( + "Python object finalization failed", + disposable: obj, innerException: e); + } + + handler.Invoke(this, new ErrorArgs() { Error = e }); @@ -267,4 +224,15 @@ private void ValidateRefCount() } #endif } + + public class FinalizationException : Exception + { + public IPyDisposable Disposable { get; } + + public FinalizationException(string message, IPyDisposable disposable, Exception innerException) + : base(message, innerException) + { + this.Disposable = disposable ?? throw new ArgumentNullException(nameof(disposable)); + } + } } diff --git a/src/runtime/importhook.cs b/src/runtime/importhook.cs index 7e4a208f5..aa3bbab6d 100644 --- a/src/runtime/importhook.cs +++ b/src/runtime/importhook.cs @@ -23,28 +23,63 @@ internal static void InitializeModuleDef() module_def = ModuleDefOffset.AllocModuleDef("clr"); } } + + internal static void ReleaseModuleDef() + { + if (module_def == IntPtr.Zero) + { + return; + } + ModuleDefOffset.FreeModuleDef(module_def); + module_def = IntPtr.Zero; + } #endif /// - /// Initialization performed on startup of the Python runtime. + /// Initialize just the __import__ hook itself. /// - internal static void Initialize() + static void InitImport() { - // Initialize the Python <--> CLR module hook. We replace the - // built-in Python __import__ with our own. This isn't ideal, - // but it provides the most "Pythonic" way of dealing with CLR - // modules (Python doesn't provide a way to emulate packages). - IntPtr dict = Runtime.PyImport_GetModuleDict(); + // We replace the built-in Python __import__ with our own: first + // look in CLR modules, then if we don't find any call the default + // Python __import__. + IntPtr builtins = Runtime.GetBuiltins(); + py_import = Runtime.PyObject_GetAttrString(builtins, "__import__"); + PythonException.ThrowIfIsNull(py_import); - IntPtr mod = Runtime.IsPython3 - ? Runtime.PyImport_ImportModule("builtins") - : Runtime.PyDict_GetItemString(dict, "__builtin__"); - - py_import = Runtime.PyObject_GetAttrString(mod, "__import__"); hook = new MethodWrapper(typeof(ImportHook), "__import__", "TernaryFunc"); - Runtime.PyObject_SetAttrString(mod, "__import__", hook.ptr); - Runtime.XDecref(hook.ptr); + int res = Runtime.PyObject_SetAttrString(builtins, "__import__", hook.ptr); + PythonException.ThrowIfIsNotZero(res); + + Runtime.XDecref(builtins); + } + + /// + /// Restore the __import__ hook. + /// + static void RestoreImport() + { + IntPtr builtins = Runtime.GetBuiltins(); + + int res = Runtime.PyObject_SetAttrString(builtins, "__import__", py_import); + PythonException.ThrowIfIsNotZero(res); + Runtime.XDecref(py_import); + py_import = IntPtr.Zero; + hook.Release(); + hook = null; + + Runtime.XDecref(builtins); + } + + /// + /// Initialization performed on startup of the Python runtime. + /// + internal static void Initialize() + { + InitImport(); + + // Initialize the clr module and tell Python about it. root = new CLRModule(); #if PYTHON3 @@ -62,6 +97,7 @@ internal static void Initialize() Runtime.XIncref(root.pyHandle); // we are using the module two times py_clr_module = root.pyHandle; // Alias handle for PY2/PY3 #endif + IntPtr dict = Runtime.PyImport_GetModuleDict(); Runtime.PyDict_SetItemString(dict, "CLR", py_clr_module); Runtime.PyDict_SetItemString(dict, "clr", py_clr_module); } @@ -72,12 +108,26 @@ internal static void Initialize() /// internal static void Shutdown() { - if (Runtime.Py_IsInitialized() != 0) + if (Runtime.Py_IsInitialized() == 0) + { + return; + } + + RestoreImport(); + + bool shouldFreeDef = Runtime.Refcount(py_clr_module) == 1; + Runtime.XDecref(py_clr_module); + py_clr_module = IntPtr.Zero; +#if PYTHON3 + if (shouldFreeDef) { - Runtime.XDecref(py_clr_module); - Runtime.XDecref(root.pyHandle); - Runtime.XDecref(py_import); + ReleaseModuleDef(); } +#endif + + Runtime.XDecref(root.pyHandle); + root = null; + CLRModule.Reset(); } /// diff --git a/src/runtime/interop.cs b/src/runtime/interop.cs index 4ae4b61e0..039feddc7 100644 --- a/src/runtime/interop.cs +++ b/src/runtime/interop.cs @@ -1,9 +1,11 @@ using System; using System.Collections; -using System.Collections.Specialized; +using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.InteropServices; using System.Reflection; using System.Text; +using System.Collections.Generic; namespace Python.Runtime { @@ -67,11 +69,47 @@ public ModulePropertyAttribute() } } + internal static class ManagedDataOffsets + { + static ManagedDataOffsets() + { + FieldInfo[] fi = typeof(ManagedDataOffsets).GetFields(BindingFlags.Static | BindingFlags.Public); + for (int i = 0; i < fi.Length; i++) + { + fi[i].SetValue(null, -(i * IntPtr.Size) - IntPtr.Size); + } - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] - internal class ObjectOffset + size = fi.Length * IntPtr.Size; + } + + public static readonly int ob_data; + public static readonly int ob_dict; + + private static int BaseOffset(IntPtr type) + { + Debug.Assert(type != IntPtr.Zero); + int typeSize = Marshal.ReadInt32(type, TypeOffset.tp_basicsize); + Debug.Assert(typeSize > 0 && typeSize <= ExceptionOffset.Size()); + return typeSize; + } + public static int DataOffset(IntPtr type) + { + return BaseOffset(type) + ob_data; + } + + public static int DictOffset(IntPtr type) + { + return BaseOffset(type) + ob_dict; + } + + public static int Size { get { return size; } } + + private static readonly int size; + } + + internal static class OriginalObjectOffsets { - static ObjectOffset() + static OriginalObjectOffsets() { int size = IntPtr.Size; var n = 0; // Py_TRACE_REFS add two pointers to PyObject_HEAD @@ -82,42 +120,58 @@ static ObjectOffset() #endif ob_refcnt = (n + 0) * size; ob_type = (n + 1) * size; - ob_dict = (n + 2) * size; - ob_data = (n + 3) * size; } - public static int magic(IntPtr ob) + public static int Size { get { return size; } } + + private static readonly int size = +#if PYTHON_WITH_PYDEBUG + 4 * IntPtr.Size; +#else + 2 * IntPtr.Size; +#endif + +#if PYTHON_WITH_PYDEBUG + public static int _ob_next; + public static int _ob_prev; +#endif + public static int ob_refcnt; + public static int ob_type; + } + + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal class ObjectOffset + { + static ObjectOffset() + { +#if PYTHON_WITH_PYDEBUG + _ob_next = OriginalObjectOffsets._ob_next; + _ob_prev = OriginalObjectOffsets._ob_prev; +#endif + ob_refcnt = OriginalObjectOffsets.ob_refcnt; + ob_type = OriginalObjectOffsets.ob_type; + + size = OriginalObjectOffsets.Size + ManagedDataOffsets.Size; + } + + public static int magic(IntPtr type) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) - { - return ExceptionOffset.ob_data; - } - return ob_data; + return ManagedDataOffsets.DataOffset(type); } - public static int DictOffset(IntPtr ob) + public static int TypeDictOffset(IntPtr type) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) - { - return ExceptionOffset.ob_dict; - } - return ob_dict; + return ManagedDataOffsets.DictOffset(type); } - public static int Size(IntPtr ob) + public static int Size(IntPtr pyType) { - if ((Runtime.PyObject_TypeCheck(ob, Exceptions.BaseException) || - (Runtime.PyType_Check(ob) && Runtime.PyType_IsSubtype(ob, Exceptions.BaseException)))) + if (IsException(pyType)) { return ExceptionOffset.Size(); } -#if PYTHON_WITH_PYDEBUG - return 6 * IntPtr.Size; -#else - return 4 * IntPtr.Size; -#endif + + return size; } #if PYTHON_WITH_PYDEBUG @@ -126,8 +180,15 @@ public static int Size(IntPtr ob) #endif public static int ob_refcnt; public static int ob_type; - private static int ob_dict; - private static int ob_data; + private static readonly int size; + + private static bool IsException(IntPtr pyObject) + { + var type = Runtime.PyObject_TYPE(pyObject); + return Runtime.PyType_IsSameAsOrSubtype(type, ofType: Exceptions.BaseException) + || Runtime.PyType_IsSameAsOrSubtype(type, ofType: Runtime.PyTypeType) + && Runtime.PyType_IsSubtype(pyObject, Exceptions.BaseException); + } } [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] @@ -136,19 +197,17 @@ internal class ExceptionOffset static ExceptionOffset() { Type type = typeof(ExceptionOffset); - FieldInfo[] fi = type.GetFields(); - int size = IntPtr.Size; + FieldInfo[] fi = type.GetFields(BindingFlags.Static | BindingFlags.Public); for (int i = 0; i < fi.Length; i++) { - fi[i].SetValue(null, (i * size) + ObjectOffset.ob_type + size); + fi[i].SetValue(null, (i * IntPtr.Size) + OriginalObjectOffsets.Size); } - } - public static int Size() - { - return ob_data + IntPtr.Size; + size = fi.Length * IntPtr.Size + OriginalObjectOffsets.Size + ManagedDataOffsets.Size; } + public static int Size() { return size; } + // PyException_HEAD // (start after PyObject_HEAD) public static int dict = 0; @@ -162,9 +221,7 @@ public static int Size() public static int suppress_context = 0; #endif - // extra c# data - public static int ob_dict; - public static int ob_data; + private static readonly int size; } @@ -227,7 +284,7 @@ public static IntPtr AllocModuleDef(string modulename) byte[] ascii = Encoding.ASCII.GetBytes(modulename); int size = name + ascii.Length + 1; IntPtr ptr = Marshal.AllocHGlobal(size); - for (int i = 0; i < m_free; i += IntPtr.Size) + for (int i = 0; i <= m_free; i += IntPtr.Size) Marshal.WriteIntPtr(ptr, i, IntPtr.Zero); Marshal.Copy(ascii, 0, (IntPtr)(ptr + name), ascii.Length); Marshal.WriteIntPtr(ptr, m_name, (IntPtr)(ptr + name)); @@ -334,7 +391,7 @@ internal class TypeFlags internal class Interop { - private static ArrayList keepAlive; + private static List keepAlive; private static Hashtable pmap; static Interop() @@ -351,8 +408,7 @@ static Interop() p[item.Name] = item; } - keepAlive = new ArrayList(); - Marshal.AllocHGlobal(IntPtr.Size); + keepAlive = new List(); pmap = new Hashtable(); pmap["tp_dealloc"] = p["DestructorFunc"]; @@ -449,7 +505,7 @@ internal static Type GetPrototype(string name) return pmap[name] as Type; } - internal static IntPtr GetThunk(MethodInfo method, string funcType = null) + internal static ThunkInfo GetThunk(MethodInfo method, string funcType = null) { Type dt; if (funcType != null) @@ -457,18 +513,15 @@ internal static IntPtr GetThunk(MethodInfo method, string funcType = null) else dt = GetPrototype(method.Name); - if (dt != null) + if (dt == null) { - IntPtr tmp = Marshal.AllocHGlobal(IntPtr.Size); - Delegate d = Delegate.CreateDelegate(dt, method); - Thunk cb = new Thunk(d); - Marshal.StructureToPtr(cb, tmp, false); - IntPtr fp = Marshal.ReadIntPtr(tmp, 0); - Marshal.FreeHGlobal(tmp); - keepAlive.Add(d); - return fp; + return ThunkInfo.Empty; } - return IntPtr.Zero; + Delegate d = Delegate.CreateDelegate(dt, method); + var info = new ThunkInfo(d); + // TODO: remove keepAlive when #958 merged, let the lifecycle of ThunkInfo transfer to caller. + keepAlive.Add(info); + return info; } [UnmanagedFunctionPointer(CallingConvention.Cdecl)] @@ -522,4 +575,22 @@ public Thunk(Delegate d) fn = d; } } + + internal class ThunkInfo + { + public readonly Delegate Target; + public readonly IntPtr Address; + + public static readonly ThunkInfo Empty = new ThunkInfo(null); + + public ThunkInfo(Delegate target) + { + if (target == null) + { + return; + } + Target = target; + Address = Marshal.GetFunctionPointerForDelegate(target); + } + } } diff --git a/src/runtime/interop38.cs b/src/runtime/interop38.cs new file mode 100644 index 000000000..9126bca6a --- /dev/null +++ b/src/runtime/interop38.cs @@ -0,0 +1,152 @@ + +// Auto-generated by geninterop.py. +// DO NOT MODIFY BY HAND. + + +#if PYTHON38 +using System; +using System.Collections; +using System.Collections.Specialized; +using System.Runtime.InteropServices; +using System.Reflection; +using System.Text; + +namespace Python.Runtime +{ + [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)] + internal class TypeOffset + { + static TypeOffset() + { + Type type = typeof(TypeOffset); + FieldInfo[] fi = type.GetFields(); + int size = IntPtr.Size; + for (int i = 0; i < fi.Length; i++) + { + fi[i].SetValue(null, i * size); + } + } + + public static int magic() + { + return ob_size; + } + + // Auto-generated from PyHeapTypeObject in Python.h + public static int ob_refcnt = 0; + public static int ob_type = 0; + public static int ob_size = 0; + public static int tp_name = 0; + public static int tp_basicsize = 0; + public static int tp_itemsize = 0; + public static int tp_dealloc = 0; + public static int tp_vectorcall_offset = 0; + public static int tp_getattr = 0; + public static int tp_setattr = 0; + public static int tp_as_async = 0; + public static int tp_repr = 0; + public static int tp_as_number = 0; + public static int tp_as_sequence = 0; + public static int tp_as_mapping = 0; + public static int tp_hash = 0; + public static int tp_call = 0; + public static int tp_str = 0; + public static int tp_getattro = 0; + public static int tp_setattro = 0; + public static int tp_as_buffer = 0; + public static int tp_flags = 0; + public static int tp_doc = 0; + public static int tp_traverse = 0; + public static int tp_clear = 0; + public static int tp_richcompare = 0; + public static int tp_weaklistoffset = 0; + public static int tp_iter = 0; + public static int tp_iternext = 0; + public static int tp_methods = 0; + public static int tp_members = 0; + public static int tp_getset = 0; + public static int tp_base = 0; + public static int tp_dict = 0; + public static int tp_descr_get = 0; + public static int tp_descr_set = 0; + public static int tp_dictoffset = 0; + public static int tp_init = 0; + public static int tp_alloc = 0; + public static int tp_new = 0; + public static int tp_free = 0; + public static int tp_is_gc = 0; + public static int tp_bases = 0; + public static int tp_mro = 0; + public static int tp_cache = 0; + public static int tp_subclasses = 0; + public static int tp_weaklist = 0; + public static int tp_del = 0; + public static int tp_version_tag = 0; + public static int tp_finalize = 0; + public static int tp_vectorcall = 0; + public static int tp_print = 0; + public static int am_await = 0; + public static int am_aiter = 0; + public static int am_anext = 0; + public static int nb_add = 0; + public static int nb_subtract = 0; + public static int nb_multiply = 0; + public static int nb_remainder = 0; + public static int nb_divmod = 0; + public static int nb_power = 0; + public static int nb_negative = 0; + public static int nb_positive = 0; + public static int nb_absolute = 0; + public static int nb_bool = 0; + public static int nb_invert = 0; + public static int nb_lshift = 0; + public static int nb_rshift = 0; + public static int nb_and = 0; + public static int nb_xor = 0; + public static int nb_or = 0; + public static int nb_int = 0; + public static int nb_reserved = 0; + public static int nb_float = 0; + public static int nb_inplace_add = 0; + public static int nb_inplace_subtract = 0; + public static int nb_inplace_multiply = 0; + public static int nb_inplace_remainder = 0; + public static int nb_inplace_power = 0; + public static int nb_inplace_lshift = 0; + public static int nb_inplace_rshift = 0; + public static int nb_inplace_and = 0; + public static int nb_inplace_xor = 0; + public static int nb_inplace_or = 0; + public static int nb_floor_divide = 0; + public static int nb_true_divide = 0; + public static int nb_inplace_floor_divide = 0; + public static int nb_inplace_true_divide = 0; + public static int nb_index = 0; + public static int nb_matrix_multiply = 0; + public static int nb_inplace_matrix_multiply = 0; + public static int mp_length = 0; + public static int mp_subscript = 0; + public static int mp_ass_subscript = 0; + public static int sq_length = 0; + public static int sq_concat = 0; + public static int sq_repeat = 0; + public static int sq_item = 0; + public static int was_sq_slice = 0; + public static int sq_ass_item = 0; + public static int was_sq_ass_slice = 0; + public static int sq_contains = 0; + public static int sq_inplace_concat = 0; + public static int sq_inplace_repeat = 0; + public static int bf_getbuffer = 0; + public static int bf_releasebuffer = 0; + public static int name = 0; + public static int ht_slots = 0; + public static int qualname = 0; + public static int ht_cached_keys = 0; + + /* here are optional user slots, followed by the members. */ + public static int members = 0; + } +} + +#endif diff --git a/src/runtime/managedtype.cs b/src/runtime/managedtype.cs index 3191da949..23f5898d1 100644 --- a/src/runtime/managedtype.cs +++ b/src/runtime/managedtype.cs @@ -33,7 +33,7 @@ internal static ManagedType GetManagedObject(IntPtr ob) { IntPtr op = tp == ob ? Marshal.ReadIntPtr(tp, TypeOffset.magic()) - : Marshal.ReadIntPtr(ob, ObjectOffset.magic(ob)); + : Marshal.ReadIntPtr(ob, ObjectOffset.magic(tp)); if (op == IntPtr.Zero) { return null; diff --git a/src/runtime/metatype.cs b/src/runtime/metatype.cs index 8853c2d5e..5af2e1a7e 100644 --- a/src/runtime/metatype.cs +++ b/src/runtime/metatype.cs @@ -266,6 +266,7 @@ private static IntPtr DoInstanceCheck(IntPtr tp, IntPtr args, bool checkType) return Runtime.PyFalse; } + Runtime.XIncref(args); using (var argsObj = new PyList(args)) { if (argsObj.Length() != 1) diff --git a/src/runtime/methodbinder.cs b/src/runtime/methodbinder.cs index 7471d5d7c..b74f21754 100644 --- a/src/runtime/methodbinder.cs +++ b/src/runtime/methodbinder.cs @@ -1,6 +1,9 @@ using System; using System.Collections; using System.Reflection; +using System.Text; +using System.Collections.Generic; +using System.Linq; namespace Python.Runtime { @@ -279,6 +282,22 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth { // loop to find match, return invoker w/ or /wo error MethodBase[] _methods = null; + + var kwargDict = new Dictionary(); + if (kw != IntPtr.Zero) + { + var pynkwargs = (int)Runtime.PyDict_Size(kw); + IntPtr keylist = Runtime.PyDict_Keys(kw); + IntPtr valueList = Runtime.PyDict_Values(kw); + for (int i = 0; i < pynkwargs; ++i) + { + var keyStr = Runtime.GetManagedString(Runtime.PyList_GetItem(keylist, i)); + kwargDict[keyStr] = Runtime.PyList_GetItem(valueList, i).DangerousGetAddress(); + } + Runtime.XDecref(keylist); + Runtime.XDecref(valueList); + } + var pynargs = (int)Runtime.PyTuple_Size(args); var isGeneric = false; if (info != null) @@ -302,11 +321,12 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth ArrayList defaultArgList; bool paramsArray; - if (!MatchesArgumentCount(pynargs, pi, out paramsArray, out defaultArgList)) { + if (!MatchesArgumentCount(pynargs, pi, kwargDict, out paramsArray, out defaultArgList)) + { continue; } var outs = 0; - var margs = TryConvertArguments(pi, paramsArray, args, pynargs, defaultArgList, + var margs = TryConvertArguments(pi, paramsArray, args, pynargs, kwargDict, defaultArgList, needsResolution: _methods.Length > 1, outs: out outs); @@ -349,20 +369,57 @@ internal Binding Bind(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, Meth return null; } + static IntPtr HandleParamsArray(IntPtr args, int arrayStart, int pyArgCount, out bool isNewReference) + { + isNewReference = false; + IntPtr op; + // for a params method, we may have a sequence or single/multiple items + // here we look to see if the item at the paramIndex is there or not + // and then if it is a sequence itself. + if ((pyArgCount - arrayStart) == 1) + { + // we only have one argument left, so we need to check it + // to see if it is a sequence or a single item + IntPtr item = Runtime.PyTuple_GetItem(args, arrayStart); + if (!Runtime.PyString_Check(item) && Runtime.PySequence_Check(item)) + { + // it's a sequence (and not a string), so we use it as the op + op = item; + } + else + { + isNewReference = true; + op = Runtime.PyTuple_GetSlice(args, arrayStart, pyArgCount); + if (item != IntPtr.Zero) + { + Runtime.XDecref(item); + } + } + } + else + { + isNewReference = true; + op = Runtime.PyTuple_GetSlice(args, arrayStart, pyArgCount); + } + return op; + } + /// - /// Attempts to convert Python argument tuple into an array of managed objects, - /// that can be passed to a method. + /// Attempts to convert Python positional argument tuple and keyword argument table + /// into an array of managed objects, that can be passed to a method. /// /// Information about expected parameters /// true, if the last parameter is a params array. /// A pointer to the Python argument tuple /// Number of arguments, passed by Python + /// Dictionary of keyword argument name to python object pointer /// A list of default values for omitted parameters /// true, if overloading resolution is required /// Returns number of output parameters /// An array of .NET arguments, that can be passed to a method. static object[] TryConvertArguments(ParameterInfo[] pi, bool paramsArray, IntPtr args, int pyArgCount, + Dictionary kwargDict, ArrayList defaultArgList, bool needsResolution, out int outs) @@ -373,7 +430,11 @@ static object[] TryConvertArguments(ParameterInfo[] pi, bool paramsArray, for (int paramIndex = 0; paramIndex < pi.Length; paramIndex++) { - if (paramIndex >= pyArgCount) + var parameter = pi[paramIndex]; + bool hasNamedParam = kwargDict.ContainsKey(parameter.Name); + bool isNewReference = false; + + if (paramIndex >= pyArgCount && !(hasNamedParam || (paramsArray && paramIndex == arrayStart))) { if (defaultArgList != null) { @@ -383,12 +444,22 @@ static object[] TryConvertArguments(ParameterInfo[] pi, bool paramsArray, continue; } - var parameter = pi[paramIndex]; - IntPtr op = (arrayStart == paramIndex) - // map remaining Python arguments to a tuple since - // the managed function accepts it - hopefully :] - ? Runtime.PyTuple_GetSlice(args, arrayStart, pyArgCount) - : Runtime.PyTuple_GetItem(args, paramIndex); + IntPtr op; + if (hasNamedParam) + { + op = kwargDict[parameter.Name]; + } + else + { + if(arrayStart == paramIndex) + { + op = HandleParamsArray(args, arrayStart, pyArgCount, out isNewReference); + } + else + { + op = Runtime.PyTuple_GetItem(args, paramIndex); + } + } bool isOut; if (!TryConvertArgument(op, parameter.ParameterType, needsResolution, out margs[paramIndex], out isOut)) @@ -396,7 +467,7 @@ static object[] TryConvertArguments(ParameterInfo[] pi, bool paramsArray, return null; } - if (arrayStart == paramIndex) + if (isNewReference) { // TODO: is this a bug? Should this happen even if the conversion fails? // GetSlice() creates a new reference but GetItem() @@ -504,29 +575,49 @@ static Type TryComputeClrArgumentType(Type parameterType, IntPtr argument, bool return clrtype; } - static bool MatchesArgumentCount(int argumentCount, ParameterInfo[] parameters, + static bool MatchesArgumentCount(int positionalArgumentCount, ParameterInfo[] parameters, + Dictionary kwargDict, out bool paramsArray, out ArrayList defaultArgList) { defaultArgList = null; var match = false; - paramsArray = false; + paramsArray = parameters.Length > 0 ? Attribute.IsDefined(parameters[parameters.Length - 1], typeof(ParamArrayAttribute)) : false; - if (argumentCount == parameters.Length) + if (positionalArgumentCount == parameters.Length) { match = true; - } else if (argumentCount < parameters.Length) + } + else if (positionalArgumentCount < parameters.Length) { + // every parameter past 'positionalArgumentCount' must have either + // a corresponding keyword argument or a default parameter match = true; defaultArgList = new ArrayList(); - for (var v = argumentCount; v < parameters.Length; v++) { - if (parameters[v].DefaultValue == DBNull.Value) { + for (var v = positionalArgumentCount; v < parameters.Length; v++) + { + if (kwargDict.ContainsKey(parameters[v].Name)) + { + // we have a keyword argument for this parameter, + // no need to check for a default parameter, but put a null + // placeholder in defaultArgList + defaultArgList.Add(null); + } + else if (parameters[v].IsOptional) + { + // IsOptional will be true if the parameter has a default value, + // or if the parameter has the [Optional] attribute specified. + // The GetDefaultValue() extension method will return the value + // to be passed in as the parameter value + defaultArgList.Add(parameters[v].GetDefaultValue()); + } + else if(!paramsArray) + { match = false; - } else { - defaultArgList.Add(parameters[v].DefaultValue); } } - } else if (argumentCount > parameters.Length && parameters.Length > 0 && + } + else if (positionalArgumentCount > parameters.Length && parameters.Length > 0 && Attribute.IsDefined(parameters[parameters.Length - 1], typeof(ParamArrayAttribute))) { // This is a `foo(params object[] bar)` style method @@ -547,6 +638,40 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i return Invoke(inst, args, kw, info, null); } + protected static void AppendArgumentTypes(StringBuilder to, IntPtr args) + { + long argCount = Runtime.PyTuple_Size(args); + to.Append("("); + for (long argIndex = 0; argIndex < argCount; argIndex++) + { + var arg = Runtime.PyTuple_GetItem(args, argIndex); + if (arg != IntPtr.Zero) + { + var type = Runtime.PyObject_Type(arg); + if (type != IntPtr.Zero) + { + try + { + var description = Runtime.PyObject_Unicode(type); + if (description != IntPtr.Zero) + { + to.Append(Runtime.GetManagedString(description)); + Runtime.XDecref(description); + } + } + finally + { + Runtime.XDecref(type); + } + } + } + + if (argIndex + 1 < argCount) + to.Append(", "); + } + to.Append(')'); + } + internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase info, MethodInfo[] methodinfo) { Binding binding = Bind(inst, args, kw, info, methodinfo); @@ -555,12 +680,15 @@ internal virtual IntPtr Invoke(IntPtr inst, IntPtr args, IntPtr kw, MethodBase i if (binding == null) { - var value = "No method matches given arguments"; + var value = new StringBuilder("No method matches given arguments"); if (methodinfo != null && methodinfo.Length > 0) { - value += $" for {methodinfo[0].Name}"; + value.Append($" for {methodinfo[0].Name}"); } - Exceptions.SetError(Exceptions.TypeError, value); + + value.Append(": "); + AppendArgumentTypes(to: value, args); + Exceptions.SetError(Exceptions.TypeError, value.ToString()); return IntPtr.Zero; } @@ -697,4 +825,33 @@ internal Binding(MethodBase info, object inst, object[] args, int outs) this.outs = outs; } } + + + static internal class ParameterInfoExtensions + { + public static object GetDefaultValue(this ParameterInfo parameterInfo) + { + // parameterInfo.HasDefaultValue is preferable but doesn't exist in .NET 4.0 + bool hasDefaultValue = (parameterInfo.Attributes & ParameterAttributes.HasDefault) == + ParameterAttributes.HasDefault; + + if (hasDefaultValue) + { + return parameterInfo.DefaultValue; + } + else + { + // [OptionalAttribute] was specified for the parameter. + // See https://stackoverflow.com/questions/3416216/optionalattribute-parameters-default-value + // for rules on determining the value to pass to the parameter + var type = parameterInfo.ParameterType; + if (type == typeof(object)) + return Type.Missing; + else if (type.IsValueType) + return Activator.CreateInstance(type); + else + return null; + } + } + } } diff --git a/src/runtime/methodwrapper.cs b/src/runtime/methodwrapper.cs index 2f3ce3ef2..bc7500dab 100644 --- a/src/runtime/methodwrapper.cs +++ b/src/runtime/methodwrapper.cs @@ -12,18 +12,21 @@ internal class MethodWrapper { public IntPtr mdef; public IntPtr ptr; + private bool _disposed = false; + + private ThunkInfo _thunk; public MethodWrapper(Type type, string name, string funcType = null) { // Turn the managed method into a function pointer - IntPtr fp = Interop.GetThunk(type.GetMethod(name), funcType); + _thunk = Interop.GetThunk(type.GetMethod(name), funcType); // Allocate and initialize a PyMethodDef structure to represent // the managed method, then create a PyCFunction. mdef = Runtime.PyMem_Malloc(4 * IntPtr.Size); - TypeManager.WriteMethodDef(mdef, name, fp, 0x0003); + TypeManager.WriteMethodDef(mdef, name, _thunk.Address, 0x0003); ptr = Runtime.PyCFunction_NewEx(mdef, IntPtr.Zero, IntPtr.Zero); } @@ -31,5 +34,21 @@ public IntPtr Call(IntPtr args, IntPtr kw) { return Runtime.PyCFunction_Call(ptr, args, kw); } + + public void Release() + { + if (_disposed) + { + return; + } + _disposed = true; + bool freeDef = Runtime.Refcount(ptr) == 1; + Runtime.XDecref(ptr); + if (freeDef && mdef != IntPtr.Zero) + { + Runtime.PyMem_Free(mdef); + mdef = IntPtr.Zero; + } + } } } diff --git a/src/runtime/moduleobject.cs b/src/runtime/moduleobject.cs index 7a45c6c81..15e4feee8 100644 --- a/src/runtime/moduleobject.cs +++ b/src/runtime/moduleobject.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using System.IO; using System.Reflection; using System.Runtime.InteropServices; @@ -53,7 +54,7 @@ public ModuleObject(string name) Runtime.XDecref(pyfilename); Runtime.XDecref(pydocstring); - Marshal.WriteIntPtr(pyHandle, ObjectOffset.DictOffset(pyHandle), dict); + Marshal.WriteIntPtr(pyHandle, ObjectOffset.TypeDictOffset(tpHandle), dict); InitializeModuleMembers(); } @@ -105,13 +106,9 @@ public ManagedType GetAttribute(string name, bool guess) // Look for a type in the current namespace. Note that this // includes types, delegates, enums, interfaces and structs. // Only public namespace members are exposed to Python. - type = AssemblyManager.LookupType(qname); + type = AssemblyManager.LookupTypes(qname).FirstOrDefault(t => t.IsPublic); if (type != null) { - if (!type.IsPublic) - { - return null; - } c = ClassManager.GetClass(type); StoreAttribute(name, c); return c; @@ -131,13 +128,9 @@ public ManagedType GetAttribute(string name, bool guess) return m; } - type = AssemblyManager.LookupType(qname); + type = AssemblyManager.LookupTypes(qname).FirstOrDefault(t => t.IsPublic); if (type != null) { - if (!type.IsPublic) - { - return null; - } c = ClassManager.GetClass(type); StoreAttribute(name, c); return c; @@ -280,7 +273,18 @@ public static IntPtr tp_getattro(IntPtr ob, IntPtr key) return self.dict; } - ManagedType attr = self.GetAttribute(name, true); + ManagedType attr = null; + + try + { + attr = self.GetAttribute(name, true); + } + catch (Exception e) + { + Exceptions.SetError(e); + return IntPtr.Zero; + } + if (attr == null) { @@ -316,6 +320,11 @@ internal class CLRModule : ModuleObject internal static bool _SuppressDocs = false; internal static bool _SuppressOverloads = false; + static CLRModule() + { + Reset(); + } + public CLRModule() : base("clr") { _namespace = string.Empty; diff --git a/src/runtime/nativecall.cs b/src/runtime/nativecall.cs index b5bf25dd7..4a7bf05c8 100644 --- a/src/runtime/nativecall.cs +++ b/src/runtime/nativecall.cs @@ -32,19 +32,21 @@ internal class NativeCall public static void Void_Call_1(IntPtr fp, IntPtr a1) { - ((Void_1_Delegate)Marshal.GetDelegateForFunctionPointer(fp, typeof(Void_1_Delegate)))(a1); + var d = Marshal.GetDelegateForFunctionPointer(fp); + d(a1); } public static IntPtr Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) { - var d = (Interop.TernaryFunc)Marshal.GetDelegateForFunctionPointer(fp, typeof(Interop.TernaryFunc)); + var d = Marshal.GetDelegateForFunctionPointer(fp); return d(a1, a2, a3); } public static int Int_Call_3(IntPtr fp, IntPtr a1, IntPtr a2, IntPtr a3) { - return ((Int_3_Delegate)Marshal.GetDelegateForFunctionPointer(fp, typeof(Int_3_Delegate)))(a1, a2, a3); + var d = Marshal.GetDelegateForFunctionPointer(fp); + return d(a1, a2, a3); } #else private static AssemblyBuilder aBuilder; diff --git a/src/runtime/platform/LibraryLoader.cs b/src/runtime/platform/LibraryLoader.cs new file mode 100644 index 000000000..a6d88cd19 --- /dev/null +++ b/src/runtime/platform/LibraryLoader.cs @@ -0,0 +1,208 @@ +using System; +using System.ComponentModel; +using System.Runtime.InteropServices; + +namespace Python.Runtime.Platform +{ + interface ILibraryLoader + { + IntPtr Load(string dllToLoad); + + IntPtr GetFunction(IntPtr hModule, string procedureName); + + void Free(IntPtr hModule); + } + + static class LibraryLoader + { + public static ILibraryLoader Get(OperatingSystemType os) + { + switch (os) + { + case OperatingSystemType.Windows: + return new WindowsLoader(); + case OperatingSystemType.Darwin: + return new DarwinLoader(); + case OperatingSystemType.Linux: + return new LinuxLoader(); + default: + throw new PlatformNotSupportedException($"This operating system ({os}) is not supported"); + } + } + } + + class LinuxLoader : ILibraryLoader + { + private static int RTLD_NOW = 0x2; + private static int RTLD_GLOBAL = 0x100; + private static IntPtr RTLD_DEFAULT = IntPtr.Zero; + private const string NativeDll = "libdl.so"; + + public IntPtr Load(string dllToLoad) + { + var filename = $"lib{dllToLoad}.so"; + ClearError(); + var res = dlopen(filename, RTLD_NOW | RTLD_GLOBAL); + if (res == IntPtr.Zero) + { + var err = GetError(); + throw new DllNotFoundException($"Could not load {filename} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); + } + + return res; + } + + public void Free(IntPtr handle) + { + dlclose(handle); + } + + public IntPtr GetFunction(IntPtr dllHandle, string name) + { + // look in the exe if dllHandle is NULL + if (dllHandle == IntPtr.Zero) + { + dllHandle = RTLD_DEFAULT; + } + + ClearError(); + IntPtr res = dlsym(dllHandle, name); + if (res == IntPtr.Zero) + { + var err = GetError(); + throw new MissingMethodException($"Failed to load symbol {name}: {err}"); + } + return res; + } + + void ClearError() + { + dlerror(); + } + + string GetError() + { + var res = dlerror(); + if (res != IntPtr.Zero) + return Marshal.PtrToStringAnsi(res); + else + return null; + } + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + public static extern IntPtr dlopen(string fileName, int flags); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym(IntPtr handle, string symbol); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose(IntPtr handle); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror(); + } + + class DarwinLoader : ILibraryLoader + { + private static int RTLD_NOW = 0x2; + private static int RTLD_GLOBAL = 0x8; + private const string NativeDll = "/usr/lib/libSystem.dylib"; + private static IntPtr RTLD_DEFAULT = new IntPtr(-2); + + public IntPtr Load(string dllToLoad) + { + var filename = $"lib{dllToLoad}.dylib"; + ClearError(); + var res = dlopen(filename, RTLD_NOW | RTLD_GLOBAL); + if (res == IntPtr.Zero) + { + var err = GetError(); + throw new DllNotFoundException($"Could not load {filename} with flags RTLD_NOW | RTLD_GLOBAL: {err}"); + } + + return res; + } + + public void Free(IntPtr handle) + { + dlclose(handle); + } + + public IntPtr GetFunction(IntPtr dllHandle, string name) + { + // look in the exe if dllHandle is NULL + if (dllHandle == IntPtr.Zero) + { + dllHandle = RTLD_DEFAULT; + } + + ClearError(); + IntPtr res = dlsym(dllHandle, name); + if (res == IntPtr.Zero) + { + var err = GetError(); + throw new MissingMethodException($"Failed to load symbol {name}: {err}"); + } + return res; + } + + void ClearError() + { + dlerror(); + } + + string GetError() + { + var res = dlerror(); + if (res != IntPtr.Zero) + return Marshal.PtrToStringAnsi(res); + else + return null; + } + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + public static extern IntPtr dlopen(String fileName, int flags); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] + private static extern IntPtr dlsym(IntPtr handle, String symbol); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern int dlclose(IntPtr handle); + + [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr dlerror(); + } + + class WindowsLoader : ILibraryLoader + { + private const string NativeDll = "kernel32.dll"; + + + public IntPtr Load(string dllToLoad) + { + var res = WindowsLoader.LoadLibrary(dllToLoad); + if (res == IntPtr.Zero) + throw new DllNotFoundException($"Could not load {dllToLoad}", new Win32Exception()); + return res; + } + + public IntPtr GetFunction(IntPtr hModule, string procedureName) + { + var res = WindowsLoader.GetProcAddress(hModule, procedureName); + if (res == IntPtr.Zero) + throw new MissingMethodException($"Failed to load symbol {procedureName}", new Win32Exception()); + return res; + } + + public void Free(IntPtr hModule) => WindowsLoader.FreeLibrary(hModule); + + [DllImport(NativeDll, SetLastError = true)] + static extern IntPtr LoadLibrary(string dllToLoad); + + [DllImport(NativeDll, SetLastError = true)] + static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); + + [DllImport(NativeDll)] + static extern bool FreeLibrary(IntPtr hModule); + } +} diff --git a/src/runtime/platform/Types.cs b/src/runtime/platform/Types.cs new file mode 100644 index 000000000..62be0e421 --- /dev/null +++ b/src/runtime/platform/Types.cs @@ -0,0 +1,23 @@ +namespace Python.Runtime.Platform +{ + public enum MachineType + { + i386, + x86_64, + armv7l, + armv8, + aarch64, + Other + }; + + /// + /// Operating system type as reported by Python. + /// + public enum OperatingSystemType + { + Windows, + Darwin, + Linux, + Other + } +} diff --git a/src/runtime/polyfill/ReflectionPolifills.cs b/src/runtime/polyfill/ReflectionPolifills.cs index a7e9c879a..b9ce78d63 100644 --- a/src/runtime/polyfill/ReflectionPolifills.cs +++ b/src/runtime/polyfill/ReflectionPolifills.cs @@ -1,12 +1,14 @@ using System; +using System.Linq; using System.Reflection; using System.Reflection.Emit; namespace Python.Runtime { -#if NETSTANDARD + [Obsolete("This API is for internal use only")] public static class ReflectionPolifills { +#if NETSTANDARD public static AssemblyBuilder DefineDynamicAssembly(this AppDomain appDomain, AssemblyName assemblyName, AssemblyBuilderAccess assemblyBuilderAccess) { return AssemblyBuilder.DefineDynamicAssembly(assemblyName, assemblyBuilderAccess); @@ -16,6 +18,19 @@ public static Type CreateType(this TypeBuilder typeBuilder) { return typeBuilder.GetTypeInfo().GetType(); } - } #endif + public static T GetCustomAttribute(this Type type) where T: Attribute + { + return type.GetCustomAttributes(typeof(T), inherit: false) + .Cast() + .SingleOrDefault(); + } + + public static T GetCustomAttribute(this Assembly assembly) where T: Attribute + { + return assembly.GetCustomAttributes(typeof(T), inherit: false) + .Cast() + .SingleOrDefault(); + } + } } diff --git a/src/runtime/pydict.cs b/src/runtime/pydict.cs index 7237d1990..7ff7a83c8 100644 --- a/src/runtime/pydict.cs +++ b/src/runtime/pydict.cs @@ -139,12 +139,20 @@ public PyObject Values() /// public PyObject Items() { - IntPtr items = Runtime.PyDict_Items(obj); - if (items == IntPtr.Zero) + var items = Runtime.PyDict_Items(this.obj); + try { - throw new PythonException(); + if (items.IsNull()) + { + throw new PythonException(); + } + + return items.MoveToPyObject(); + } + finally + { + items.Dispose(); } - return new PyObject(items); } diff --git a/src/runtime/pyint.cs b/src/runtime/pyint.cs index f6911d9d7..217cf7e20 100644 --- a/src/runtime/pyint.cs +++ b/src/runtime/pyint.cs @@ -62,7 +62,7 @@ public PyInt(int value) /// Creates a new Python int from a uint32 value. /// [CLSCompliant(false)] - public PyInt(uint value) : base(IntPtr.Zero) + public PyInt(uint value) { obj = Runtime.PyInt_FromInt64(value); Runtime.CheckExceptionOccurred(); @@ -75,7 +75,7 @@ public PyInt(uint value) : base(IntPtr.Zero) /// /// Creates a new Python int from an int64 value. /// - public PyInt(long value) : base(IntPtr.Zero) + public PyInt(long value) { obj = Runtime.PyInt_FromInt64(value); Runtime.CheckExceptionOccurred(); @@ -89,7 +89,7 @@ public PyInt(long value) : base(IntPtr.Zero) /// Creates a new Python int from a uint64 value. /// [CLSCompliant(false)] - public PyInt(ulong value) : base(IntPtr.Zero) + public PyInt(ulong value) { obj = Runtime.PyInt_FromInt64((long)value); Runtime.CheckExceptionOccurred(); diff --git a/src/runtime/pylist.cs b/src/runtime/pylist.cs index b22d9d51f..347cc3000 100644 --- a/src/runtime/pylist.cs +++ b/src/runtime/pylist.cs @@ -120,7 +120,7 @@ public static PyList AsList(PyObject value) /// public void Append(PyObject item) { - int r = Runtime.PyList_Append(obj, item.obj); + int r = Runtime.PyList_Append(this.Reference, item.obj); if (r < 0) { throw new PythonException(); @@ -135,7 +135,7 @@ public void Append(PyObject item) /// public void Insert(int index, PyObject item) { - int r = Runtime.PyList_Insert(obj, index, item.obj); + int r = Runtime.PyList_Insert(this.Reference, index, item.obj); if (r < 0) { throw new PythonException(); @@ -151,7 +151,7 @@ public void Insert(int index, PyObject item) /// public void Reverse() { - int r = Runtime.PyList_Reverse(obj); + int r = Runtime.PyList_Reverse(this.Reference); if (r < 0) { throw new PythonException(); @@ -167,7 +167,7 @@ public void Reverse() /// public void Sort() { - int r = Runtime.PyList_Sort(obj); + int r = Runtime.PyList_Sort(this.Reference); if (r < 0) { throw new PythonException(); diff --git a/src/runtime/pyobject.cs b/src/runtime/pyobject.cs index c86504802..699ebf873 100644 --- a/src/runtime/pyobject.cs +++ b/src/runtime/pyobject.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Dynamic; +using System.Linq; using System.Linq.Expressions; namespace Python.Runtime @@ -29,8 +30,8 @@ public class PyObject : DynamicObject, IEnumerable, IPyDisposable #endif protected internal IntPtr obj = IntPtr.Zero; - private bool disposed = false; - private bool _finalized = false; + + internal BorrowedReference Reference => new BorrowedReference(obj); /// /// PyObject Constructor @@ -43,7 +44,26 @@ public class PyObject : DynamicObject, IEnumerable, IPyDisposable /// public PyObject(IntPtr ptr) { + if (ptr == IntPtr.Zero) throw new ArgumentNullException(nameof(ptr)); + obj = ptr; + Finalizer.Instance.ThrottledCollect(); +#if TRACE_ALLOC + Traceback = new StackTrace(1); +#endif + } + + /// + /// Creates new pointing to the same object as + /// the . Increments refcount, allowing + /// to have ownership over its own reference. + /// + internal PyObject(BorrowedReference reference) + { + if (reference.IsNull) throw new ArgumentNullException(nameof(reference)); + + obj = Runtime.SelfIncRef(reference.DangerousGetAddress()); + Finalizer.Instance.ThrottledCollect(); #if TRACE_ALLOC Traceback = new StackTrace(1); #endif @@ -51,9 +71,10 @@ public PyObject(IntPtr ptr) // Protected default constructor to allow subclasses to manage // initialization in different ways as appropriate. - + [Obsolete("Please, always use PyObject(*Reference)")] protected PyObject() { + Finalizer.Instance.ThrottledCollect(); #if TRACE_ALLOC Traceback = new StackTrace(1); #endif @@ -67,12 +88,6 @@ protected PyObject() { return; } - if (_finalized || disposed) - { - return; - } - // Prevent a infinity loop by calling GC.WaitForPendingFinalizers - _finalized = true; Finalizer.Instance.AddFinalizedObject(this); } @@ -91,7 +106,8 @@ public IntPtr Handle /// - /// FromManagedObject Method + /// Gets raw Python proxy for this object (bypasses all conversions, + /// except null <==> None) /// /// /// Given an arbitrary managed object, return a Python instance that @@ -162,17 +178,41 @@ public T As() /// protected virtual void Dispose(bool disposing) { - if (!disposed) + if (this.obj == IntPtr.Zero) { - if (Runtime.Py_IsInitialized() > 0 && !Runtime.IsFinalizing) + return; + } + + if (Runtime.Py_IsInitialized() == 0) + throw new InvalidOperationException("Python runtime must be initialized"); + + if (!Runtime.IsFinalizing) + { + long refcount = Runtime.Refcount(this.obj); + Debug.Assert(refcount > 0, "Object refcount is 0 or less"); + + if (refcount == 1) { - IntPtr gs = PythonEngine.AcquireLock(); - Runtime.XDecref(obj); - obj = IntPtr.Zero; - PythonEngine.ReleaseLock(gs); + Runtime.PyErr_Fetch(out var errType, out var errVal, out var traceback); + + try + { + Runtime.XDecref(this.obj); + Runtime.CheckExceptionOccurred(); + } + finally + { + // Python requires finalizers to preserve exception: + // https://docs.python.org/3/extending/newtypes.html#finalization-and-de-allocation + Runtime.PyErr_Restore(errType, errVal, traceback); + } + } + else + { + Runtime.XDecref(this.obj); } - disposed = true; } + this.obj = IntPtr.Zero; } public void Dispose() @@ -209,6 +249,8 @@ public PyObject GetPythonType() /// public bool TypeCheck(PyObject typeOrClass) { + if (typeOrClass == null) throw new ArgumentNullException(nameof(typeOrClass)); + return Runtime.PyObject_TypeCheck(obj, typeOrClass.obj); } @@ -221,6 +263,8 @@ public bool TypeCheck(PyObject typeOrClass) /// public bool HasAttr(string name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + return Runtime.PyObject_HasAttrString(obj, name) != 0; } @@ -234,6 +278,8 @@ public bool HasAttr(string name) /// public bool HasAttr(PyObject name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + return Runtime.PyObject_HasAttr(obj, name.obj) != 0; } @@ -247,6 +293,8 @@ public bool HasAttr(PyObject name) /// public PyObject GetAttr(string name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + IntPtr op = Runtime.PyObject_GetAttrString(obj, name); if (op == IntPtr.Zero) { @@ -257,7 +305,7 @@ public PyObject GetAttr(string name) /// - /// GetAttr Method + /// GetAttr Method. Returns fallback value if getting attribute fails for any reason. /// /// /// Returns the named attribute of the Python object, or the given @@ -265,6 +313,8 @@ public PyObject GetAttr(string name) /// public PyObject GetAttr(string name, PyObject _default) { + if (name == null) throw new ArgumentNullException(nameof(name)); + IntPtr op = Runtime.PyObject_GetAttrString(obj, name); if (op == IntPtr.Zero) { @@ -285,6 +335,8 @@ public PyObject GetAttr(string name, PyObject _default) /// public PyObject GetAttr(PyObject name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + IntPtr op = Runtime.PyObject_GetAttr(obj, name.obj); if (op == IntPtr.Zero) { @@ -304,6 +356,8 @@ public PyObject GetAttr(PyObject name) /// public PyObject GetAttr(PyObject name, PyObject _default) { + if (name == null) throw new ArgumentNullException(nameof(name)); + IntPtr op = Runtime.PyObject_GetAttr(obj, name.obj); if (op == IntPtr.Zero) { @@ -323,6 +377,9 @@ public PyObject GetAttr(PyObject name, PyObject _default) /// public void SetAttr(string name, PyObject value) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (value == null) throw new ArgumentNullException(nameof(value)); + int r = Runtime.PyObject_SetAttrString(obj, name, value.obj); if (r < 0) { @@ -341,6 +398,9 @@ public void SetAttr(string name, PyObject value) /// public void SetAttr(PyObject name, PyObject value) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (value == null) throw new ArgumentNullException(nameof(value)); + int r = Runtime.PyObject_SetAttr(obj, name.obj, value.obj); if (r < 0) { @@ -358,6 +418,8 @@ public void SetAttr(PyObject name, PyObject value) /// public void DelAttr(string name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + int r = Runtime.PyObject_SetAttrString(obj, name, IntPtr.Zero); if (r < 0) { @@ -376,6 +438,8 @@ public void DelAttr(string name) /// public void DelAttr(PyObject name) { + if (name == null) throw new ArgumentNullException(nameof(name)); + int r = Runtime.PyObject_SetAttr(obj, name.obj, IntPtr.Zero); if (r < 0) { @@ -394,6 +458,8 @@ public void DelAttr(PyObject name) /// public virtual PyObject GetItem(PyObject key) { + if (key == null) throw new ArgumentNullException(nameof(key)); + IntPtr op = Runtime.PyObject_GetItem(obj, key.obj); if (op == IntPtr.Zero) { @@ -413,6 +479,8 @@ public virtual PyObject GetItem(PyObject key) /// public virtual PyObject GetItem(string key) { + if (key == null) throw new ArgumentNullException(nameof(key)); + using (var pyKey = new PyString(key)) { return GetItem(pyKey); @@ -447,6 +515,9 @@ public virtual PyObject GetItem(int index) /// public virtual void SetItem(PyObject key, PyObject value) { + if (key == null) throw new ArgumentNullException(nameof(key)); + if (value == null) throw new ArgumentNullException(nameof(value)); + int r = Runtime.PyObject_SetItem(obj, key.obj, value.obj); if (r < 0) { @@ -465,6 +536,9 @@ public virtual void SetItem(PyObject key, PyObject value) /// public virtual void SetItem(string key, PyObject value) { + if (key == null) throw new ArgumentNullException(nameof(key)); + if (value == null) throw new ArgumentNullException(nameof(value)); + using (var pyKey = new PyString(key)) { SetItem(pyKey, value); @@ -482,6 +556,8 @@ public virtual void SetItem(string key, PyObject value) /// public virtual void SetItem(int index, PyObject value) { + if (value == null) throw new ArgumentNullException(nameof(value)); + using (var pyindex = new PyInt(index)) { SetItem(pyindex, value); @@ -499,6 +575,8 @@ public virtual void SetItem(int index, PyObject value) /// public virtual void DelItem(PyObject key) { + if (key == null) throw new ArgumentNullException(nameof(key)); + int r = Runtime.PyObject_DelItem(obj, key.obj); if (r < 0) { @@ -517,6 +595,8 @@ public virtual void DelItem(PyObject key) /// public virtual void DelItem(string key) { + if (key == null) throw new ArgumentNullException(nameof(key)); + using (var pyKey = new PyString(key)) { DelItem(pyKey); @@ -639,10 +719,13 @@ public IEnumerator GetEnumerator() /// /// /// Invoke the callable object with the given arguments, passed as a - /// PyObject[]. A PythonException is raised if the invokation fails. + /// PyObject[]. A PythonException is raised if the invocation fails. /// public PyObject Invoke(params PyObject[] args) { + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + var t = new PyTuple(args); IntPtr r = Runtime.PyObject_Call(obj, t.obj, IntPtr.Zero); t.Dispose(); @@ -659,10 +742,12 @@ public PyObject Invoke(params PyObject[] args) /// /// /// Invoke the callable object with the given arguments, passed as a - /// Python tuple. A PythonException is raised if the invokation fails. + /// Python tuple. A PythonException is raised if the invocation fails. /// public PyObject Invoke(PyTuple args) { + if (args == null) throw new ArgumentNullException(nameof(args)); + IntPtr r = Runtime.PyObject_Call(obj, args.obj, IntPtr.Zero); if (r == IntPtr.Zero) { @@ -677,12 +762,15 @@ public PyObject Invoke(PyTuple args) /// /// /// Invoke the callable object with the given positional and keyword - /// arguments. A PythonException is raised if the invokation fails. + /// arguments. A PythonException is raised if the invocation fails. /// public PyObject Invoke(PyObject[] args, PyDict kw) { + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + var t = new PyTuple(args); - IntPtr r = Runtime.PyObject_Call(obj, t.obj, kw != null ? kw.obj : IntPtr.Zero); + IntPtr r = Runtime.PyObject_Call(obj, t.obj, kw?.obj ?? IntPtr.Zero); t.Dispose(); if (r == IntPtr.Zero) { @@ -697,11 +785,13 @@ public PyObject Invoke(PyObject[] args, PyDict kw) /// /// /// Invoke the callable object with the given positional and keyword - /// arguments. A PythonException is raised if the invokation fails. + /// arguments. A PythonException is raised if the invocation fails. /// public PyObject Invoke(PyTuple args, PyDict kw) { - IntPtr r = Runtime.PyObject_Call(obj, args.obj, kw != null ? kw.obj : IntPtr.Zero); + if (args == null) throw new ArgumentNullException(nameof(args)); + + IntPtr r = Runtime.PyObject_Call(obj, args.obj, kw?.obj ?? IntPtr.Zero); if (r == IntPtr.Zero) { throw new PythonException(); @@ -715,10 +805,14 @@ public PyObject Invoke(PyTuple args, PyDict kw) /// /// /// Invoke the named method of the object with the given arguments. - /// A PythonException is raised if the invokation is unsuccessful. + /// A PythonException is raised if the invocation is unsuccessful. /// public PyObject InvokeMethod(string name, params PyObject[] args) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + PyObject method = GetAttr(name); PyObject result = method.Invoke(args); method.Dispose(); @@ -731,10 +825,51 @@ public PyObject InvokeMethod(string name, params PyObject[] args) /// /// /// Invoke the named method of the object with the given arguments. - /// A PythonException is raised if the invokation is unsuccessful. + /// A PythonException is raised if the invocation is unsuccessful. /// public PyObject InvokeMethod(string name, PyTuple args) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + + PyObject method = GetAttr(name); + PyObject result = method.Invoke(args); + method.Dispose(); + return result; + } + + /// + /// InvokeMethod Method + /// + /// + /// Invoke the named method of the object with the given arguments. + /// A PythonException is raised if the invocation is unsuccessful. + /// + public PyObject InvokeMethod(PyObject name, params PyObject[] args) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + + PyObject method = GetAttr(name); + PyObject result = method.Invoke(args); + method.Dispose(); + return result; + } + + + /// + /// InvokeMethod Method + /// + /// + /// Invoke the named method of the object with the given arguments. + /// A PythonException is raised if the invocation is unsuccessful. + /// + public PyObject InvokeMethod(PyObject name, PyTuple args) + { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + PyObject method = GetAttr(name); PyObject result = method.Invoke(args); method.Dispose(); @@ -748,10 +883,14 @@ public PyObject InvokeMethod(string name, PyTuple args) /// /// Invoke the named method of the object with the given arguments /// and keyword arguments. Keyword args are passed as a PyDict object. - /// A PythonException is raised if the invokation is unsuccessful. + /// A PythonException is raised if the invocation is unsuccessful. /// public PyObject InvokeMethod(string name, PyObject[] args, PyDict kw) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + if (args.Contains(null)) throw new ArgumentNullException(); + PyObject method = GetAttr(name); PyObject result = method.Invoke(args, kw); method.Dispose(); @@ -765,10 +904,13 @@ public PyObject InvokeMethod(string name, PyObject[] args, PyDict kw) /// /// Invoke the named method of the object with the given arguments /// and keyword arguments. Keyword args are passed as a PyDict object. - /// A PythonException is raised if the invokation is unsuccessful. + /// A PythonException is raised if the invocation is unsuccessful. /// public PyObject InvokeMethod(string name, PyTuple args, PyDict kw) { + if (name == null) throw new ArgumentNullException(nameof(name)); + if (args == null) throw new ArgumentNullException(nameof(args)); + PyObject method = GetAttr(name); PyObject result = method.Invoke(args, kw); method.Dispose(); @@ -785,6 +927,8 @@ public PyObject InvokeMethod(string name, PyTuple args, PyDict kw) /// public bool IsInstance(PyObject typeOrClass) { + if (typeOrClass == null) throw new ArgumentNullException(nameof(typeOrClass)); + int r = Runtime.PyObject_IsInstance(obj, typeOrClass.obj); if (r < 0) { @@ -804,6 +948,8 @@ public bool IsInstance(PyObject typeOrClass) /// public bool IsSubclass(PyObject typeOrClass) { + if (typeOrClass == null) throw new ArgumentNullException(nameof(typeOrClass)); + int r = Runtime.PyObject_IsSubclass(obj, typeOrClass.obj); if (r < 0) { @@ -852,6 +998,10 @@ public bool IsTrue() return Runtime.PyObject_IsTrue(obj) != 0; } + /// + /// Return true if the object is None + /// + public bool IsNone() => CheckNone(this) == null; /// /// Dir Method diff --git a/src/runtime/pyscope.cs b/src/runtime/pyscope.cs index 4008ce29a..8738824f5 100644 --- a/src/runtime/pyscope.cs +++ b/src/runtime/pyscope.cs @@ -278,11 +278,19 @@ public PyObject Eval(string code, PyDict locals = null) Check(); IntPtr _locals = locals == null ? variables : locals.obj; var flag = (IntPtr)Runtime.Py_eval_input; - IntPtr ptr = Runtime.PyRun_String( + + NewReference reference = Runtime.PyRun_String( code, flag, variables, _locals ); - Runtime.CheckExceptionOccurred(); - return new PyObject(ptr); + try + { + Runtime.CheckExceptionOccurred(); + return reference.MoveToPyObject(); + } + finally + { + reference.Dispose(); + } } /// @@ -316,15 +324,22 @@ public void Exec(string code, PyDict locals = null) private void Exec(string code, IntPtr _globals, IntPtr _locals) { var flag = (IntPtr)Runtime.Py_file_input; - IntPtr ptr = Runtime.PyRun_String( + NewReference reference = Runtime.PyRun_String( code, flag, _globals, _locals ); - Runtime.CheckExceptionOccurred(); - if (ptr != Runtime.PyNone) + + try { - throw new PythonException(); + Runtime.CheckExceptionOccurred(); + if (!reference.IsNone()) + { + throw new PythonException(); + } + } + finally + { + reference.Dispose(); } - Runtime.XDecref(ptr); } /// diff --git a/src/runtime/pythonengine.cs b/src/runtime/pythonengine.cs index c1b663d22..11fc88cd4 100644 --- a/src/runtime/pythonengine.cs +++ b/src/runtime/pythonengine.cs @@ -80,6 +80,7 @@ public static string PythonHome } set { + // this value is null in the beginning Marshal.FreeHGlobal(_pythonHome); _pythonHome = UcsMarshaler.Py3UnicodePy2StringtoPtr(value); Runtime.Py_SetPythonHome(_pythonHome); @@ -130,6 +131,16 @@ public static string Compiler get { return Marshal.PtrToStringAnsi(Runtime.Py_GetCompiler()); } } + /// + /// Set the NoSiteFlag to disable loading the site module. + /// Must be called before Initialize. + /// https://docs.python.org/3/c-api/init.html#c.Py_NoSiteFlag + /// + public static void SetNoSiteFlag() + { + Runtime.SetNoSiteFlag(); + } + public static int RunSimpleString(string code) { return Runtime.PyRun_SimpleString(code); @@ -318,6 +329,8 @@ public static void Shutdown() ExecuteShutdownHandlers(); + PyObjectConversions.Reset(); + initialized = false; } } @@ -493,7 +506,7 @@ public static PyObject ReloadModule(PyObject module) /// public static PyObject ModuleFromString(string name, string code) { - IntPtr c = Runtime.Py_CompileString(code, "none", (IntPtr)257); + IntPtr c = Runtime.Py_CompileString(code, "none", (int)RunFlagType.File); Runtime.CheckExceptionOccurred(); IntPtr m = Runtime.PyImport_ExecCodeModule(name, c); Runtime.CheckExceptionOccurred(); @@ -502,7 +515,7 @@ public static PyObject ModuleFromString(string name, string code) public static PyObject Compile(string code, string filename = "", RunFlagType mode = RunFlagType.File) { - var flag = (IntPtr)mode; + var flag = (int)mode; IntPtr ptr = Runtime.Py_CompileString(code, filename, flag); Runtime.CheckExceptionOccurred(); return new PyObject(ptr); @@ -531,12 +544,13 @@ public static PyObject Eval(string code, IntPtr? globals = null, IntPtr? locals /// public static void Exec(string code, IntPtr? globals = null, IntPtr? locals = null) { - PyObject result = RunString(code, globals, locals, RunFlagType.File); - if (result.obj != Runtime.PyNone) + using (PyObject result = RunString(code, globals, locals, RunFlagType.File)) { - throw new PythonException(); + if (result.obj != Runtime.PyNone) + { + throw new PythonException(); + } } - result.Dispose(); } @@ -582,13 +596,20 @@ internal static PyObject RunString(string code, IntPtr? globals, IntPtr? locals, try { - IntPtr result = Runtime.PyRun_String( + NewReference result = Runtime.PyRun_String( code, (IntPtr)flag, globals.Value, locals.Value ); - Runtime.CheckExceptionOccurred(); + try + { + Runtime.CheckExceptionOccurred(); - return new PyObject(result); + return result.MoveToPyObject(); + } + finally + { + result.Dispose(); + } } finally { @@ -600,7 +621,7 @@ internal static PyObject RunString(string code, IntPtr? globals, IntPtr? locals, } } - public enum RunFlagType + public enum RunFlagType : int { Single = 256, File = 257, /* Py_file_input */ @@ -633,7 +654,8 @@ public static PyScope CreateScope(string name) public class GILState : IDisposable { - private IntPtr state; + private readonly IntPtr state; + private bool isDisposed; internal GILState() { @@ -642,8 +664,11 @@ internal GILState() public void Dispose() { + if (this.isDisposed) return; + PythonEngine.ReleaseLock(state); GC.SuppressFinalize(this); + this.isDisposed = true; } ~GILState() @@ -744,11 +769,14 @@ public static void With(PyObject obj, Action Body) catch (PythonException e) { ex = e; - type = ex.PyType; - val = ex.PyValue; - traceBack = ex.PyTB; + type = ex.PyType.Coalesce(type); + val = ex.PyValue.Coalesce(val); + traceBack = ex.PyTB.Coalesce(traceBack); } + Runtime.XIncref(type); + Runtime.XIncref(val); + Runtime.XIncref(traceBack); var exitResult = obj.InvokeMethod("__exit__", new PyObject(type), new PyObject(val), new PyObject(traceBack)); if (ex != null && !exitResult.IsTrue()) throw ex; diff --git a/src/runtime/pythonexception.cs b/src/runtime/pythonexception.cs index 295a63b3d..8efdccc91 100644 --- a/src/runtime/pythonexception.cs +++ b/src/runtime/pythonexception.cs @@ -1,4 +1,6 @@ using System; +using System.Runtime.CompilerServices; +using System.Text; namespace Python.Runtime { @@ -20,7 +22,7 @@ public class PythonException : System.Exception, IPyDisposable public PythonException() { IntPtr gs = PythonEngine.AcquireLock(); - Runtime.PyErr_Fetch(ref _pyType, ref _pyValue, ref _pyTB); + Runtime.PyErr_Fetch(out _pyType, out _pyValue, out _pyTB); if (_pyType != IntPtr.Zero && _pyValue != IntPtr.Zero) { string type; @@ -144,6 +146,47 @@ public string PythonTypeName get { return _pythonTypeName; } } + /// + /// Formats this PythonException object into a message as would be printed + /// out via the Python console. See traceback.format_exception + /// + public string Format() + { + string res; + IntPtr gs = PythonEngine.AcquireLock(); + try + { + if (_pyTB != IntPtr.Zero && _pyType != IntPtr.Zero && _pyValue != IntPtr.Zero) + { + Runtime.XIncref(_pyType); + Runtime.XIncref(_pyValue); + Runtime.XIncref(_pyTB); + using (PyObject pyType = new PyObject(_pyType)) + using (PyObject pyValue = new PyObject(_pyValue)) + using (PyObject pyTB = new PyObject(_pyTB)) + using (PyObject tb_mod = PythonEngine.ImportModule("traceback")) + { + var buffer = new StringBuilder(); + var values = tb_mod.InvokeMethod("format_exception", pyType, pyValue, pyTB); + foreach (PyObject val in values) + { + buffer.Append(val.ToString()); + } + res = buffer.ToString(); + } + } + else + { + res = StackTrace; + } + } + finally + { + PythonEngine.ReleaseLock(gs); + } + return res; + } + /// /// Dispose Method /// @@ -160,12 +203,23 @@ public void Dispose() if (Runtime.Py_IsInitialized() > 0 && !Runtime.IsFinalizing) { IntPtr gs = PythonEngine.AcquireLock(); - Runtime.XDecref(_pyType); - Runtime.XDecref(_pyValue); + if (_pyType != IntPtr.Zero) + { + Runtime.XDecref(_pyType); + _pyType= IntPtr.Zero; + } + + if (_pyValue != IntPtr.Zero) + { + Runtime.XDecref(_pyValue); + _pyValue = IntPtr.Zero; + } + // XXX Do we ever get TraceBack? // if (_pyTB != IntPtr.Zero) { Runtime.XDecref(_pyTB); + _pyTB = IntPtr.Zero; } PythonEngine.ReleaseLock(gs); } @@ -190,5 +244,21 @@ public static bool Matches(IntPtr ob) { return Runtime.PyErr_ExceptionMatches(ob) != 0; } + + public static void ThrowIfIsNull(IntPtr ob) + { + if (ob == IntPtr.Zero) + { + throw new PythonException(); + } + } + + public static void ThrowIfIsNotZero(int value) + { + if (value != 0) + { + throw new PythonException(); + } + } } } diff --git a/src/runtime/resources/clr.py b/src/runtime/resources/clr.py index ddb0d94e8..78fae0d3a 100644 --- a/src/runtime/resources/clr.py +++ b/src/runtime/resources/clr.py @@ -2,7 +2,7 @@ Code in this module gets loaded into the main clr module. """ -__version__ = "2.4.0" +__version__ = "2.5.0" class clrproperty(object): diff --git a/src/runtime/runtime.cs b/src/runtime/runtime.cs index 7623200e0..8cf24ba70 100644 --- a/src/runtime/runtime.cs +++ b/src/runtime/runtime.cs @@ -1,103 +1,15 @@ +using System.Reflection.Emit; using System; +using System.Diagnostics.Contracts; using System.Runtime.InteropServices; using System.Security; using System.Text; using System.Threading; using System.Collections.Generic; +using Python.Runtime.Platform; namespace Python.Runtime { - [SuppressUnmanagedCodeSecurity] - internal static class NativeMethods - { -#if MONO_LINUX || MONO_OSX -#if NETSTANDARD - private static int RTLD_NOW = 0x2; -#if MONO_LINUX - private static int RTLD_GLOBAL = 0x100; - private static IntPtr RTLD_DEFAULT = IntPtr.Zero; - private const string NativeDll = "libdl.so"; - public static IntPtr LoadLibrary(string fileName) - { - return dlopen($"lib{fileName}.so", RTLD_NOW | RTLD_GLOBAL); - } -#elif MONO_OSX - private static int RTLD_GLOBAL = 0x8; - private const string NativeDll = "/usr/lib/libSystem.dylib"; - private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - - public static IntPtr LoadLibrary(string fileName) - { - return dlopen($"lib{fileName}.dylib", RTLD_NOW | RTLD_GLOBAL); - } -#endif -#else - private static int RTLD_NOW = 0x2; - private static int RTLD_SHARED = 0x20; -#if MONO_OSX - private static IntPtr RTLD_DEFAULT = new IntPtr(-2); - private const string NativeDll = "__Internal"; -#elif MONO_LINUX - private static IntPtr RTLD_DEFAULT = IntPtr.Zero; - private const string NativeDll = "libdl.so"; -#endif - - public static IntPtr LoadLibrary(string fileName) - { - return dlopen(fileName, RTLD_NOW | RTLD_SHARED); - } -#endif - - - public static void FreeLibrary(IntPtr handle) - { - dlclose(handle); - } - - public static IntPtr GetProcAddress(IntPtr dllHandle, string name) - { - // look in the exe if dllHandle is NULL - if (dllHandle == IntPtr.Zero) - { - dllHandle = RTLD_DEFAULT; - } - - // clear previous errors if any - dlerror(); - IntPtr res = dlsym(dllHandle, name); - IntPtr errPtr = dlerror(); - if (errPtr != IntPtr.Zero) - { - throw new Exception("dlsym: " + Marshal.PtrToStringAnsi(errPtr)); - } - return res; - } - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - public static extern IntPtr dlopen(String fileName, int flags); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl, CharSet = CharSet.Ansi)] - private static extern IntPtr dlsym(IntPtr handle, String symbol); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int dlclose(IntPtr handle); - - [DllImport(NativeDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr dlerror(); -#else // Windows - private const string NativeDll = "kernel32.dll"; - - [DllImport(NativeDll)] - public static extern IntPtr LoadLibrary(string dllToLoad); - - [DllImport(NativeDll)] - public static extern IntPtr GetProcAddress(IntPtr hModule, string procedureName); - - [DllImport(NativeDll)] - public static extern bool FreeLibrary(IntPtr hModule); -#endif - } - /// /// Encapsulates the low-level Python C API. Note that it is /// the responsibility of the caller to have acquired the GIL @@ -153,8 +65,11 @@ public class Runtime #elif PYTHON37 internal const string _pyversion = "3.7"; internal const string _pyver = "37"; +#elif PYTHON38 + internal const string _pyversion = "3.8"; + internal const string _pyver = "38"; #else -#error You must define one of PYTHON34 to PYTHON37 or PYTHON27 +#error You must define one of PYTHON34 to PYTHON38 or PYTHON27 #endif #if MONO_LINUX || MONO_OSX // Linux/macOS use dotted version string @@ -192,22 +107,11 @@ public class Runtime internal static object IsFinalizingLock = new object(); internal static bool IsFinalizing; - internal static bool Is32Bit = IntPtr.Size == 4; + internal static bool Is32Bit => IntPtr.Size == 4; // .NET core: System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(OSPlatform.Windows) internal static bool IsWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; - /// - /// Operating system type as reported by Python. - /// - public enum OperatingSystemType - { - Windows, - Darwin, - Linux, - Other - } - static readonly Dictionary OperatingSystemTypeMapping = new Dictionary() { { "Windows", OperatingSystemType.Windows }, @@ -215,22 +119,16 @@ public enum OperatingSystemType { "Linux", OperatingSystemType.Linux }, }; - /// - /// Gets the operating system as reported by python's platform.system(). - /// - public static OperatingSystemType OperatingSystem { get; private set; } + [Obsolete] + public static string OperatingSystemName => OperatingSystem.ToString(); + + [Obsolete] + public static string MachineName => Machine.ToString(); /// /// Gets the operating system as reported by python's platform.system(). /// - public static string OperatingSystemName { get; private set; } - - public enum MachineType - { - i386, - x86_64, - Other - }; + public static OperatingSystemType OperatingSystem { get; private set; } /// /// Map lower-case version of the python machine name to the processor @@ -247,6 +145,9 @@ public enum MachineType ["amd64"] = MachineType.x86_64, ["x64"] = MachineType.x86_64, ["em64t"] = MachineType.x86_64, + ["armv7l"] = MachineType.armv7l, + ["armv8"] = MachineType.armv8, + ["aarch64"] = MachineType.aarch64, }; /// @@ -254,11 +155,6 @@ public enum MachineType /// public static MachineType Machine { get; private set; }/* set in Initialize using python's platform.machine */ - /// - /// Gets the machine architecture as reported by python's platform.machine(). - /// - public static string MachineName { get; private set; } - internal static bool IsPython2 = pyversionnumber < 30; internal static bool IsPython3 = pyversionnumber >= 30; @@ -269,6 +165,8 @@ public enum MachineType /// internal static readonly Encoding PyEncoding = _UCS == 2 ? Encoding.Unicode : Encoding.UTF32; + private static PyReferenceCollection _pyRefs = new PyReferenceCollection(); + /// /// Initialize the runtime... /// @@ -287,7 +185,6 @@ internal static void Initialize(bool initSigs = false) IsFinalizing = false; - CLRModule.Reset(); GenericUtil.Reset(); PyScopeManager.Reset(); ClassManager.Reset(); @@ -295,122 +192,139 @@ internal static void Initialize(bool initSigs = false) TypeManager.Reset(); IntPtr op; - IntPtr dict; - if (IsPython3) - { - op = PyImport_ImportModule("builtins"); - dict = PyObject_GetAttrString(op, "__dict__"); - } - else // Python2 { - dict = PyImport_GetModuleDict(); - op = PyDict_GetItemString(dict, "__builtin__"); + var builtins = GetBuiltins(); + SetPyMember(ref PyNotImplemented, PyObject_GetAttrString(builtins, "NotImplemented"), + () => PyNotImplemented = IntPtr.Zero); + + SetPyMember(ref PyBaseObjectType, PyObject_GetAttrString(builtins, "object"), + () => PyBaseObjectType = IntPtr.Zero); + + SetPyMember(ref PyNone, PyObject_GetAttrString(builtins, "None"), + () => PyNone = IntPtr.Zero); + SetPyMember(ref PyTrue, PyObject_GetAttrString(builtins, "True"), + () => PyTrue = IntPtr.Zero); + SetPyMember(ref PyFalse, PyObject_GetAttrString(builtins, "False"), + () => PyFalse = IntPtr.Zero); + + SetPyMember(ref PyBoolType, PyObject_Type(PyTrue), + () => PyBoolType = IntPtr.Zero); + SetPyMember(ref PyNoneType, PyObject_Type(PyNone), + () => PyNoneType = IntPtr.Zero); + SetPyMember(ref PyTypeType, PyObject_Type(PyNoneType), + () => PyTypeType = IntPtr.Zero); + + op = PyObject_GetAttrString(builtins, "len"); + SetPyMember(ref PyMethodType, PyObject_Type(op), + () => PyMethodType = IntPtr.Zero); + XDecref(op); + + // For some arcane reason, builtins.__dict__.__setitem__ is *not* + // a wrapper_descriptor, even though dict.__setitem__ is. + // + // object.__init__ seems safe, though. + op = PyObject_GetAttrString(PyBaseObjectType, "__init__"); + SetPyMember(ref PyWrapperDescriptorType, PyObject_Type(op), + () => PyWrapperDescriptorType = IntPtr.Zero); + XDecref(op); + + SetPyMember(ref PySuper_Type, PyObject_GetAttrString(builtins, "super"), + () => PySuper_Type = IntPtr.Zero); + + XDecref(builtins); } - PyNotImplemented = PyObject_GetAttrString(op, "NotImplemented"); - PyBaseObjectType = PyObject_GetAttrString(op, "object"); - - PyModuleType = PyObject_Type(op); - PyNone = PyObject_GetAttrString(op, "None"); - PyTrue = PyObject_GetAttrString(op, "True"); - PyFalse = PyObject_GetAttrString(op, "False"); - - PyBoolType = PyObject_Type(PyTrue); - PyNoneType = PyObject_Type(PyNone); - PyTypeType = PyObject_Type(PyNoneType); - - op = PyObject_GetAttrString(dict, "keys"); - PyMethodType = PyObject_Type(op); - XDecref(op); - - // For some arcane reason, builtins.__dict__.__setitem__ is *not* - // a wrapper_descriptor, even though dict.__setitem__ is. - // - // object.__init__ seems safe, though. - op = PyObject_GetAttrString(PyBaseObjectType, "__init__"); - PyWrapperDescriptorType = PyObject_Type(op); - XDecref(op); - -#if PYTHON3 - XDecref(dict); -#endif op = PyString_FromString("string"); - PyStringType = PyObject_Type(op); + SetPyMember(ref PyStringType, PyObject_Type(op), + () => PyStringType = IntPtr.Zero); XDecref(op); op = PyUnicode_FromString("unicode"); - PyUnicodeType = PyObject_Type(op); + SetPyMember(ref PyUnicodeType, PyObject_Type(op), + () => PyUnicodeType = IntPtr.Zero); XDecref(op); #if PYTHON3 op = PyBytes_FromString("bytes"); - PyBytesType = PyObject_Type(op); + SetPyMember(ref PyBytesType, PyObject_Type(op), + () => PyBytesType = IntPtr.Zero); XDecref(op); #endif op = PyTuple_New(0); - PyTupleType = PyObject_Type(op); + SetPyMember(ref PyTupleType, PyObject_Type(op), + () => PyTupleType = IntPtr.Zero); XDecref(op); op = PyList_New(0); - PyListType = PyObject_Type(op); + SetPyMember(ref PyListType, PyObject_Type(op), + () => PyListType = IntPtr.Zero); XDecref(op); op = PyDict_New(); - PyDictType = PyObject_Type(op); + SetPyMember(ref PyDictType, PyObject_Type(op), + () => PyDictType = IntPtr.Zero); XDecref(op); op = PyInt_FromInt32(0); - PyIntType = PyObject_Type(op); + SetPyMember(ref PyIntType, PyObject_Type(op), + () => PyIntType = IntPtr.Zero); XDecref(op); op = PyLong_FromLong(0); - PyLongType = PyObject_Type(op); + SetPyMember(ref PyLongType, PyObject_Type(op), + () => PyLongType = IntPtr.Zero); XDecref(op); op = PyFloat_FromDouble(0); - PyFloatType = PyObject_Type(op); + SetPyMember(ref PyFloatType, PyObject_Type(op), + () => PyFloatType = IntPtr.Zero); XDecref(op); -#if PYTHON3 +#if !PYTHON2 PyClassType = IntPtr.Zero; PyInstanceType = IntPtr.Zero; -#elif PYTHON2 - IntPtr s = PyString_FromString("_temp"); - IntPtr d = PyDict_New(); +#else + { + IntPtr s = PyString_FromString("_temp"); + IntPtr d = PyDict_New(); - IntPtr c = PyClass_New(IntPtr.Zero, d, s); - PyClassType = PyObject_Type(c); + IntPtr c = PyClass_New(IntPtr.Zero, d, s); + SetPyMember(ref PyClassType, PyObject_Type(c), + () => PyClassType = IntPtr.Zero); - IntPtr i = PyInstance_New(c, IntPtr.Zero, IntPtr.Zero); - PyInstanceType = PyObject_Type(i); + IntPtr i = PyInstance_New(c, IntPtr.Zero, IntPtr.Zero); + SetPyMember(ref PyInstanceType, PyObject_Type(i), + () => PyInstanceType = IntPtr.Zero); - XDecref(s); - XDecref(i); - XDecref(c); - XDecref(d); + XDecref(s); + XDecref(i); + XDecref(c); + XDecref(d); + } #endif Error = new IntPtr(-1); + // Initialize data about the platform we're running on. We need + // this for the type manager and potentially other details. Must + // happen after caching the python types, above. + InitializePlatformData(); + IntPtr dllLocal = IntPtr.Zero; + var loader = LibraryLoader.Get(OperatingSystem); if (_PythonDll != "__Internal") { - dllLocal = NativeMethods.LoadLibrary(_PythonDll); + dllLocal = loader.Load(_PythonDll); } - _PyObject_NextNotImplemented = NativeMethods.GetProcAddress(dllLocal, "_PyObject_NextNotImplemented"); + _PyObject_NextNotImplemented = loader.GetFunction(dllLocal, "_PyObject_NextNotImplemented"); + PyModuleType = loader.GetFunction(dllLocal, "PyModule_Type"); -#if !(MONO_LINUX || MONO_OSX) if (dllLocal != IntPtr.Zero) { - NativeMethods.FreeLibrary(dllLocal); + loader.Free(dllLocal); } -#endif - // Initialize data about the platform we're running on. We need - // this for the type manager and potentially other details. Must - // happen after caching the python types, above. - InitializePlatformData(); // Initialize modules that depend on the runtime class. AssemblyManager.Initialize(); @@ -423,7 +337,7 @@ internal static void Initialize(bool initSigs = false) string rtdir = RuntimeEnvironment.GetRuntimeDirectory(); IntPtr path = PySys_GetObject("path"); IntPtr item = PyString_FromString(rtdir); - PyList_Append(path, item); + PyList_Append(new BorrowedReference(path), item); XDecref(item); AssemblyManager.UpdatePath(); } @@ -438,6 +352,7 @@ internal static void Initialize(bool initSigs = false) /// private static void InitializePlatformData() { +#if !NETSTANDARD IntPtr op; IntPtr fn; IntPtr platformModule = PyImport_ImportModule("platform"); @@ -445,13 +360,13 @@ private static void InitializePlatformData() fn = PyObject_GetAttrString(platformModule, "system"); op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - OperatingSystemName = GetManagedString(op); + string operatingSystemName = GetManagedString(op); XDecref(op); XDecref(fn); fn = PyObject_GetAttrString(platformModule, "machine"); op = PyObject_Call(fn, emptyTuple, IntPtr.Zero); - MachineName = GetManagedString(op); + string machineName = GetManagedString(op); XDecref(op); XDecref(fn); @@ -461,18 +376,47 @@ private static void InitializePlatformData() // Now convert the strings into enum values so we can do switch // statements rather than constant parsing. OperatingSystemType OSType; - if (!OperatingSystemTypeMapping.TryGetValue(OperatingSystemName, out OSType)) + if (!OperatingSystemTypeMapping.TryGetValue(operatingSystemName, out OSType)) { OSType = OperatingSystemType.Other; } OperatingSystem = OSType; MachineType MType; - if (!MachineTypeMapping.TryGetValue(MachineName.ToLower(), out MType)) + if (!MachineTypeMapping.TryGetValue(machineName.ToLower(), out MType)) { MType = MachineType.Other; } Machine = MType; +#else + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + OperatingSystem = OperatingSystemType.Linux; + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + OperatingSystem = OperatingSystemType.Darwin; + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + OperatingSystem = OperatingSystemType.Windows; + else + OperatingSystem = OperatingSystemType.Other; + + switch (RuntimeInformation.ProcessArchitecture) + { + case Architecture.X86: + Machine = MachineType.i386; + break; + case Architecture.X64: + Machine = MachineType.x86_64; + break; + case Architecture.Arm: + Machine = MachineType.armv7l; + break; + case Architecture.Arm64: + Machine = MachineType.aarch64; + break; + default: + Machine = MachineType.Other; + break; + } +#endif } internal static void Shutdown() @@ -481,6 +425,9 @@ internal static void Shutdown() Exceptions.Shutdown(); ImportHook.Shutdown(); Finalizer.Shutdown(); + // TOOD: PyCLRMetaType's release operation still in #958 + PyCLRMetaType = IntPtr.Zero; + ResetPyMembers(); Py_Finalize(); } @@ -494,6 +441,19 @@ internal static int AtExit() return 0; } + private static void SetPyMember(ref IntPtr obj, IntPtr value, Action onRelease) + { + // XXX: For current usages, value should not be null. + PythonException.ThrowIfIsNull(value); + obj = value; + _pyRefs.Add(value, onRelease); + } + + private static void ResetPyMembers() + { + _pyRefs.Release(); + } + internal static IntPtr Py_single_input = (IntPtr)256; internal static IntPtr Py_file_input = (IntPtr)257; internal static IntPtr Py_eval_input = (IntPtr)258; @@ -502,6 +462,7 @@ internal static int AtExit() internal static IntPtr PyModuleType; internal static IntPtr PyClassType; internal static IntPtr PyInstanceType; + internal static IntPtr PySuper_Type; internal static IntPtr PyCLRMetaType; internal static IntPtr PyMethodType; internal static IntPtr PyWrapperDescriptorType; @@ -518,6 +479,8 @@ internal static int AtExit() internal static IntPtr PyNoneType; internal static IntPtr PyTypeType; + internal static IntPtr Py_NoSiteFlag; + #if PYTHON3 internal static IntPtr PyBytesType; #endif @@ -536,6 +499,16 @@ internal static int AtExit() internal static IntPtr PyNone; internal static IntPtr Error; + public static PyObject None + { + get + { + var none = Runtime.PyNone; + Runtime.XIncref(none); + return new PyObject(none); + } + } + /// /// Check if any Python Exceptions occurred. /// If any exist throw new PythonException. @@ -666,6 +639,15 @@ internal static unsafe void XIncref(IntPtr op) #endif } + /// + /// Increase Python's ref counter for the given object, and get the object back. + /// + internal static IntPtr SelfIncRef(IntPtr op) + { + XIncref(op); + return op; + } + internal static unsafe void XDecref(IntPtr op) { #if PYTHON_WITH_PYDEBUG || NETSTANDARD @@ -703,6 +685,7 @@ internal static unsafe void XDecref(IntPtr op) #endif } + [Pure] internal static unsafe long Refcount(IntPtr op) { var p = (void*)op; @@ -856,13 +839,45 @@ public static extern int Py_Main( internal static extern int PyRun_SimpleString(string code); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyRun_String(string code, IntPtr st, IntPtr globals, IntPtr locals); +#if PYTHON2 + internal static extern NewReference PyRun_String(string code, IntPtr st, IntPtr globals, IntPtr locals); +#else + internal static extern NewReference PyRun_String([MarshalAs(UnmanagedType.CustomMarshaler, MarshalTypeRef = typeof(Utf8Marshaler))] string code, IntPtr st, IntPtr globals, IntPtr locals); +#endif [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyEval_EvalCode(IntPtr co, IntPtr globals, IntPtr locals); +#if PYTHON2 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr Py_CompileString(string code, string file, IntPtr tok); + internal static extern IntPtr Py_CompileString(string code, string file, int start); +#else + /// + /// Return value: New reference. + /// This is a simplified interface to Py_CompileStringFlags() below, leaving flags set to NULL. + /// + internal static IntPtr Py_CompileString(string str, string file, int start) + { + return Py_CompileStringFlags(str, file, start, IntPtr.Zero); + } + + /// + /// Return value: New reference. + /// This is a simplified interface to Py_CompileStringExFlags() below, with optimize set to -1. + /// + internal static IntPtr Py_CompileStringFlags(string str, string file, int start, IntPtr flags) + { + return Py_CompileStringExFlags(str, file, start, flags, -1); + } + + /// + /// Return value: New reference. + /// Like Py_CompileStringObject(), but filename is a byte string decoded from the filesystem encoding(os.fsdecode()). + /// + /// + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr Py_CompileStringExFlags(string str, string file, int start, IntPtr flags, int optimize); +#endif [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyImport_ExecCodeModule(string name, IntPtr code); @@ -873,8 +888,10 @@ public static extern int Py_Main( [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyCFunction_Call(IntPtr func, IntPtr args, IntPtr kw); +#if PYTHON2 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyClass_New(IntPtr bases, IntPtr dict, IntPtr name); +#endif [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyInstance_New(IntPtr cls, IntPtr args, IntPtr kw); @@ -1032,7 +1049,7 @@ internal static int PyObject_Compare(IntPtr value1, IntPtr value2) internal static long PyObject_Size(IntPtr pointer) { - return (long) _PyObject_Size(pointer); + return (long)_PyObject_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyObject_Size")] @@ -1116,10 +1133,6 @@ internal static IntPtr PyInt_FromInt64(long value) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyLong_FromString")] internal static extern IntPtr PyInt_FromString(string value, IntPtr end, int radix); - - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, - EntryPoint = "PyLong_GetMax")] - internal static extern int PyInt_GetMax(); #elif PYTHON2 [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr PyInt_FromLong(IntPtr value); @@ -1142,8 +1155,21 @@ internal static bool PyLong_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyLong_FromLong(long value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyLong_FromUnsignedLong(uint value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_FromUnsignedLong")] + internal static extern IntPtr PyLong_FromUnsignedLong32(uint value); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_FromUnsignedLong")] + internal static extern IntPtr PyLong_FromUnsignedLong64(ulong value); + + internal static IntPtr PyLong_FromUnsignedLong(object value) + { + if (Is32Bit || IsWindows) + return PyLong_FromUnsignedLong32(Convert.ToUInt32(value)); + else + return PyLong_FromUnsignedLong64(Convert.ToUInt64(value)); + } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyLong_FromDouble(double value); @@ -1160,8 +1186,21 @@ internal static bool PyLong_Check(IntPtr ob) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int PyLong_AsLong(IntPtr value); - [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern uint PyLong_AsUnsignedLong(IntPtr value); + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_AsUnsignedLong")] + internal static extern uint PyLong_AsUnsignedLong32(IntPtr value); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, + EntryPoint = "PyLong_AsUnsignedLong")] + internal static extern ulong PyLong_AsUnsignedLong64(IntPtr value); + + internal static object PyLong_AsUnsignedLong(IntPtr value) + { + if (Is32Bit || IsWindows) + return PyLong_AsUnsignedLong32(value); + else + return PyLong_AsUnsignedLong64(value); + } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern long PyLong_AsLongLong(IntPtr value); @@ -1316,7 +1355,7 @@ internal static int PySequence_DelSlice(IntPtr pointer, long i1, long i2) internal static long PySequence_Size(IntPtr pointer) { - return (long) _PySequence_Size(pointer); + return (long)_PySequence_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Size")] @@ -1341,7 +1380,7 @@ internal static IntPtr PySequence_Repeat(IntPtr pointer, long count) internal static long PySequence_Count(IntPtr pointer, IntPtr value) { - return (long) _PySequence_Count(pointer, value); + return (long)_PySequence_Count(pointer, value); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PySequence_Count")] @@ -1384,7 +1423,7 @@ internal static IntPtr PyString_FromString(string value) internal static long PyBytes_Size(IntPtr op) { - return (long) _PyBytes_Size(op); + return (long)_PyBytes_Size(op); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyBytes_Size")] @@ -1414,6 +1453,10 @@ internal static IntPtr PyUnicode_FromStringAndSize(IntPtr value, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] private static extern IntPtr PyUnicode_FromStringAndSize(IntPtr value, IntPtr size); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern IntPtr PyUnicode_AsUTF8(IntPtr unicode); + #elif PYTHON2 internal static IntPtr PyString_FromStringAndSize(string value, long size) { @@ -1516,6 +1559,8 @@ internal static IntPtr PyUnicode_FromString(string s) return PyUnicode_FromUnicode(s, s.Length); } + internal static string GetManagedString(in BorrowedReference borrowedReference) + => GetManagedString(borrowedReference.DangerousGetAddress()); /// /// Function to access the internal PyUnicode/PyString object and /// convert it to a managed string with the correct encoding. @@ -1598,7 +1643,7 @@ internal static bool PyDict_Check(IntPtr ob) internal static extern IntPtr PyDict_Values(IntPtr pointer); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern IntPtr PyDict_Items(IntPtr pointer); + internal static extern NewReference PyDict_Items(IntPtr pointer); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyDict_Copy(IntPtr pointer); @@ -1611,7 +1656,7 @@ internal static bool PyDict_Check(IntPtr ob) internal static long PyDict_Size(IntPtr pointer) { - return (long) _PyDict_Size(pointer); + return (long)_PyDict_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyDict_Size")] @@ -1638,13 +1683,13 @@ internal static IntPtr PyList_New(long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyList_AsTuple(IntPtr pointer); - internal static IntPtr PyList_GetItem(IntPtr pointer, long index) + internal static BorrowedReference PyList_GetItem(IntPtr pointer, long index) { return PyList_GetItem(pointer, new IntPtr(index)); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr PyList_GetItem(IntPtr pointer, IntPtr index); + private static extern BorrowedReference PyList_GetItem(IntPtr pointer, IntPtr index); internal static int PyList_SetItem(IntPtr pointer, long index, IntPtr value) { @@ -1654,22 +1699,22 @@ internal static int PyList_SetItem(IntPtr pointer, long index, IntPtr value) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] private static extern int PyList_SetItem(IntPtr pointer, IntPtr index, IntPtr value); - internal static int PyList_Insert(IntPtr pointer, long index, IntPtr value) + internal static int PyList_Insert(BorrowedReference pointer, long index, IntPtr value) { return PyList_Insert(pointer, new IntPtr(index), value); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - private static extern int PyList_Insert(IntPtr pointer, IntPtr index, IntPtr value); + private static extern int PyList_Insert(BorrowedReference pointer, IntPtr index, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyList_Append(IntPtr pointer, IntPtr value); + internal static extern int PyList_Append(BorrowedReference pointer, IntPtr value); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyList_Reverse(IntPtr pointer); + internal static extern int PyList_Reverse(BorrowedReference pointer); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern int PyList_Sort(IntPtr pointer); + internal static extern int PyList_Sort(BorrowedReference pointer); internal static IntPtr PyList_GetSlice(IntPtr pointer, long start, long end) { @@ -1689,7 +1734,7 @@ internal static int PyList_SetSlice(IntPtr pointer, long start, long end, IntPtr internal static long PyList_Size(IntPtr pointer) { - return (long) _PyList_Size(pointer); + return (long)_PyList_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyList_Size")] @@ -1738,7 +1783,7 @@ internal static IntPtr PyTuple_GetSlice(IntPtr pointer, long start, long end) internal static long PyTuple_Size(IntPtr pointer) { - return (long) _PyTuple_Size(pointer); + return (long)_PyTuple_Size(pointer); } [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl, EntryPoint = "PyTuple_Size")] @@ -1789,6 +1834,9 @@ internal static bool PyIter_Check(IntPtr pointer) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyImport_Import(IntPtr name); + /// + /// Return value: New reference. + /// [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyImport_ImportModule(string name); @@ -1845,6 +1893,11 @@ internal static bool PyObject_TypeCheck(IntPtr ob, IntPtr tp) return (t == tp) || PyType_IsSubtype(t, tp); } + internal static bool PyType_IsSameAsOrSubtype(IntPtr type, IntPtr ofType) + { + return (type == ofType) || PyType_IsSubtype(type, ofType); + } + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern IntPtr PyType_GenericNew(IntPtr type, IntPtr args, IntPtr kw); @@ -1937,7 +1990,7 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) internal static extern IntPtr PyErr_Occurred(); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] - internal static extern void PyErr_Fetch(ref IntPtr ob, ref IntPtr val, ref IntPtr tb); + internal static extern void PyErr_Fetch(out IntPtr ob, out IntPtr val, out IntPtr tb); [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_Restore(IntPtr ob, IntPtr val, IntPtr tb); @@ -1948,6 +2001,15 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern void PyErr_Print(); + //==================================================================== + // Cell API + //==================================================================== + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern NewReference PyCell_Get(BorrowedReference cell); + + [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] + internal static extern int PyCell_Set(BorrowedReference cell, IntPtr value); //==================================================================== // Miscellaneous @@ -1964,5 +2026,61 @@ internal static IntPtr PyMem_Realloc(IntPtr ptr, long size) [DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)] internal static extern int Py_MakePendingCalls(); + + internal static void SetNoSiteFlag() + { + var loader = LibraryLoader.Get(OperatingSystem); + + IntPtr dllLocal = _PythonDll != "__Internal" + ? loader.Load(_PythonDll) + : IntPtr.Zero; + + try + { + Py_NoSiteFlag = loader.GetFunction(dllLocal, "Py_NoSiteFlag"); + Marshal.WriteInt32(Py_NoSiteFlag, 1); + } + finally + { + if (dllLocal != IntPtr.Zero) + { + loader.Free(dllLocal); + } + } + } + + /// + /// Return value: New reference. + /// + internal static IntPtr GetBuiltins() + { + return IsPython3 ? PyImport_ImportModule("builtins") + : PyImport_ImportModule("__builtin__"); + } + } + + + class PyReferenceCollection + { + private List> _actions = new List>(); + + /// + /// Record obj's address to release the obj in the future, + /// obj must alive before calling Release. + /// + public void Add(IntPtr ob, Action onRelease) + { + _actions.Add(new KeyValuePair(ob, onRelease)); + } + + public void Release() + { + foreach (var item in _actions) + { + Runtime.XDecref(item.Key); + item.Value?.Invoke(); + } + _actions.Clear(); + } } } diff --git a/src/runtime/slots/mp_length.cs b/src/runtime/slots/mp_length.cs new file mode 100644 index 000000000..b0a2e8d79 --- /dev/null +++ b/src/runtime/slots/mp_length.cs @@ -0,0 +1,50 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; + +namespace Python.Runtime.Slots +{ + internal static class mp_length_slot + { + /// + /// Implements __len__ for classes that implement ICollection + /// (this includes any IList implementer or Array subclass) + /// + public static int mp_length(IntPtr ob) + { + var co = ManagedType.GetManagedObject(ob) as CLRObject; + if (co == null) + { + Exceptions.RaiseTypeError("invalid object"); + } + + // first look for ICollection implementation directly + if (co.inst is ICollection c) + { + return c.Count; + } + + Type clrType = co.inst.GetType(); + + // now look for things that implement ICollection directly (non-explicitly) + PropertyInfo p = clrType.GetProperty("Count"); + if (p != null && clrType.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))) + { + return (int)p.GetValue(co.inst, null); + } + + // finally look for things that implement the interface explicitly + var iface = clrType.GetInterfaces().FirstOrDefault(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>)); + if (iface != null) + { + p = iface.GetProperty(nameof(ICollection.Count)); + return (int)p.GetValue(co.inst, null); + } + + Exceptions.SetError(Exceptions.TypeError, $"object of type '{clrType.Name}' has no len()"); + return -1; + } + } +} diff --git a/src/runtime/typemanager.cs b/src/runtime/typemanager.cs index d19c8737f..3df873eb5 100644 --- a/src/runtime/typemanager.cs +++ b/src/runtime/typemanager.cs @@ -1,11 +1,15 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Linq; using System.Reflection; using System.Runtime.InteropServices; +using Python.Runtime.Platform; +using Python.Runtime.Slots; namespace Python.Runtime { + /// /// The TypeManager class is responsible for building binary-compatible /// Python type objects that are implemented in managed code. @@ -83,7 +87,7 @@ internal static IntPtr CreateType(Type impl) // Set tp_basicsize to the size of our managed instance objects. Marshal.WriteIntPtr(type, TypeOffset.tp_basicsize, (IntPtr)ob_size); - var offset = (IntPtr)ObjectOffset.DictOffset(type); + var offset = (IntPtr)ObjectOffset.TypeDictOffset(type); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, offset); InitializeSlots(type, impl); @@ -121,7 +125,6 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) IntPtr base_ = IntPtr.Zero; int ob_size = ObjectOffset.Size(Runtime.PyTypeType); - int tp_dictoffset = ObjectOffset.DictOffset(Runtime.PyTypeType); // XXX Hack, use a different base class for System.Exception // Python 2.5+ allows new style class exceptions but they *must* @@ -129,9 +132,10 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) if (typeof(Exception).IsAssignableFrom(clrType)) { ob_size = ObjectOffset.Size(Exceptions.Exception); - tp_dictoffset = ObjectOffset.DictOffset(Exceptions.Exception); } + int tp_dictoffset = ob_size + ManagedDataOffsets.ob_dict; + if (clrType == typeof(Exception)) { base_ = Exceptions.Exception; @@ -151,6 +155,13 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) Marshal.WriteIntPtr(type, TypeOffset.tp_itemsize, IntPtr.Zero); Marshal.WriteIntPtr(type, TypeOffset.tp_dictoffset, (IntPtr)tp_dictoffset); + // add a __len__ slot for inheritors of ICollection and ICollection<> + if (typeof(ICollection).IsAssignableFrom(clrType) || clrType.GetInterfaces().Any(x => x.IsGenericType && x.GetGenericTypeDefinition() == typeof(ICollection<>))) + { + InitializeSlot(type, TypeOffset.mp_length, typeof(mp_length_slot).GetMethod(nameof(mp_length_slot.mp_length))); + } + + // we want to do this after the slot stuff above in case the class itself implements a slot method InitializeSlots(type, impl.GetType()); if (base_ != IntPtr.Zero) @@ -191,6 +202,12 @@ internal static IntPtr CreateType(ManagedType impl, Type clrType) return type; } + static void InitializeSlot(IntPtr type, int slotOffset, MethodInfo method) + { + var thunk = Interop.GetThunk(method); + Marshal.WriteIntPtr(type, slotOffset, thunk.Address); + } + internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr py_dict) { // Utility to create a subtype of a managed type with the ability for the @@ -263,6 +280,14 @@ internal static IntPtr CreateSubType(IntPtr py_name, IntPtr py_base_type, IntPtr IntPtr cls_dict = Marshal.ReadIntPtr(py_type, TypeOffset.tp_dict); Runtime.PyDict_Update(cls_dict, py_dict); + // Update the __classcell__ if it exists + var cell = new BorrowedReference(Runtime.PyDict_GetItemString(cls_dict, "__classcell__")); + if (!cell.IsNull) + { + Runtime.PyCell_Set(cell, py_type); + Runtime.PyDict_DelItemString(cls_dict, "__classcell__"); + } + return py_type; } catch (Exception e) @@ -307,17 +332,11 @@ internal static IntPtr CreateMetaType(Type impl) Marshal.WriteIntPtr(type, TypeOffset.tp_base, py_type); Runtime.XIncref(py_type); - // Copy gc and other type slots from the base Python metatype. - - CopySlot(py_type, type, TypeOffset.tp_basicsize); - CopySlot(py_type, type, TypeOffset.tp_itemsize); - - CopySlot(py_type, type, TypeOffset.tp_dictoffset); - CopySlot(py_type, type, TypeOffset.tp_weaklistoffset); - - CopySlot(py_type, type, TypeOffset.tp_traverse); - CopySlot(py_type, type, TypeOffset.tp_clear); - CopySlot(py_type, type, TypeOffset.tp_is_gc); + // Slots will inherit from TypeType, it's not neccesary for setting them. + // Inheried slots: + // tp_basicsize, tp_itemsize, + // tp_dictoffset, tp_weaklistoffset, + // tp_traverse, tp_clear, tp_is_gc, etc. // Override type slots with those of the managed implementation. @@ -333,16 +352,18 @@ internal static IntPtr CreateMetaType(Type impl) // 4 int-ptrs in size. IntPtr mdef = Runtime.PyMem_Malloc(3 * 4 * IntPtr.Size); IntPtr mdefStart = mdef; + ThunkInfo thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc"); mdef = WriteMethodDef( mdef, "__instancecheck__", - Interop.GetThunk(typeof(MetaType).GetMethod("__instancecheck__"), "BinaryFunc") + thunkInfo.Address ); + thunkInfo = Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc"); mdef = WriteMethodDef( mdef, "__subclasscheck__", - Interop.GetThunk(typeof(MetaType).GetMethod("__subclasscheck__"), "BinaryFunc") + thunkInfo.Address ); // FIXME: mdef is not used @@ -413,12 +434,8 @@ internal static IntPtr AllocateTypeObject(string name) // the Python version of the type name - otherwise we'd have to // allocate the tp_name and would have no way to free it. #if PYTHON3 - // For python3 we leak two objects. One for the ASCII representation - // required for tp_name, and another for the Unicode representation - // for ht_name. - IntPtr temp = Runtime.PyBytes_FromString(name); - IntPtr raw = Runtime.PyBytes_AS_STRING(temp); - temp = Runtime.PyUnicode_FromString(name); + IntPtr temp = Runtime.PyUnicode_FromString(name); + IntPtr raw = Runtime.PyUnicode_AsUTF8(temp); #elif PYTHON2 IntPtr temp = Runtime.PyString_FromString(name); IntPtr raw = Runtime.PyString_AsString(temp); @@ -504,14 +521,14 @@ public static NativeCode Active { get { - switch(Runtime.Machine) + switch (Runtime.Machine) { - case Runtime.MachineType.i386: + case MachineType.i386: return I386; - case Runtime.MachineType.x86_64: + case MachineType.x86_64: return X86_64; default: - throw new NotImplementedException($"No support for {Runtime.MachineName}"); + return null; } } } @@ -603,12 +620,14 @@ int MAP_ANONYMOUS { switch (Runtime.OperatingSystem) { - case Runtime.OperatingSystemType.Darwin: + case OperatingSystemType.Darwin: return 0x1000; - case Runtime.OperatingSystemType.Linux: + case OperatingSystemType.Linux: return 0x20; default: - throw new NotImplementedException($"mmap is not supported on {Runtime.OperatingSystemName}"); + throw new NotImplementedException( + $"mmap is not supported on {Runtime.OperatingSystem}" + ); } } } @@ -636,13 +655,15 @@ internal static IMemoryMapper CreateMemoryMapper() { switch (Runtime.OperatingSystem) { - case Runtime.OperatingSystemType.Darwin: - case Runtime.OperatingSystemType.Linux: + case OperatingSystemType.Darwin: + case OperatingSystemType.Linux: return new UnixMemoryMapper(); - case Runtime.OperatingSystemType.Windows: + case OperatingSystemType.Windows: return new WindowsMemoryMapper(); default: - throw new NotImplementedException($"No support for {Runtime.OperatingSystemName}"); + throw new NotImplementedException( + $"No support for {Runtime.OperatingSystem}" + ); } } @@ -668,7 +689,7 @@ internal static void InitializeNativeCodePage() Marshal.Copy(NativeCode.Active.Code, 0, NativeCodePage, codeLength); mapper.SetReadExec(NativeCodePage, codeLength); } -#endregion + #endregion /// /// Given a newly allocated Python type object and a managed Type that @@ -703,7 +724,8 @@ internal static void InitializeSlots(IntPtr type, Type impl) continue; } - InitializeSlot(type, Interop.GetThunk(method), name); + var thunkInfo = Interop.GetThunk(method); + InitializeSlot(type, thunkInfo.Address, name); seen.Add(name); } @@ -711,21 +733,40 @@ internal static void InitializeSlots(IntPtr type, Type impl) impl = impl.BaseType; } - // See the TestDomainReload test: there was a crash related to - // the gc-related slots. They always return 0 or 1 because we don't - // really support gc: + var native = NativeCode.Active; + + // The garbage collection related slots always have to return 1 or 0 + // since .NET objects don't take part in Python's gc: // tp_traverse (returns 0) // tp_clear (returns 0) // tp_is_gc (returns 1) - // We can't do without: python really wants those slots to exist. - // We can't implement those in C# because the application domain - // can be shut down and the memory released. - InitializeNativeCodePage(); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return0, "tp_traverse"); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return0, "tp_clear"); - InitializeSlot(type, NativeCodePage + NativeCode.Active.Return1, "tp_is_gc"); + // These have to be defined, though, so by default we fill these with + // static C# functions from this class. + + var ret0 = Interop.GetThunk(((Func)Return0).Method).Address; + var ret1 = Interop.GetThunk(((Func)Return1).Method).Address; + + if (native != null) + { + // If we want to support domain reload, the C# implementation + // cannot be used as the assembly may get released before + // CPython calls these functions. Instead, for amd64 and x86 we + // load them into a separate code page that is leaked + // intentionally. + InitializeNativeCodePage(); + ret1 = NativeCodePage + native.Return1; + ret0 = NativeCodePage + native.Return0; + } + + InitializeSlot(type, ret0, "tp_traverse"); + InitializeSlot(type, ret0, "tp_clear"); + InitializeSlot(type, ret1, "tp_is_gc"); } + static int Return1(IntPtr _) => 1; + + static int Return0(IntPtr _) => 0; + /// /// Helper for InitializeSlots. /// diff --git a/src/testing/Python.Test.15.csproj b/src/testing/Python.Test.15.csproj index da20ed2ef..0e19adf91 100644 --- a/src/testing/Python.Test.15.csproj +++ b/src/testing/Python.Test.15.csproj @@ -7,7 +7,7 @@ Python.Test Python.Test Python.Test - 2.4.0 + 2.5.0 bin\ false $(OutputPath)\$(AssemblyName).xml diff --git a/src/testing/Python.Test.csproj b/src/testing/Python.Test.csproj index 27639ed5a..63526c060 100644 --- a/src/testing/Python.Test.csproj +++ b/src/testing/Python.Test.csproj @@ -91,6 +91,9 @@ + + + @@ -111,4 +114,4 @@ - \ No newline at end of file + diff --git a/src/testing/ReprTest.cs b/src/testing/ReprTest.cs new file mode 100644 index 000000000..48e93683a --- /dev/null +++ b/src/testing/ReprTest.cs @@ -0,0 +1,108 @@ +using System; +using System.Text; + +namespace Python.Test +{ + /// + /// Supports repr unit tests. + /// + public class ReprTest + { + public class Point + { + public Point(double x, double y) + { + X = x; + Y = y; + } + + public double X { get; set; } + public double Y { get; set; } + + public override string ToString() + { + return base.ToString() + ": X=" + X.ToString() + ", Y=" + Y.ToString(); + } + + public string __repr__() + { + return "Point(" + X.ToString() + "," + Y.ToString() + ")"; + } + } + + public class Foo + { + public string __repr__() + { + return "I implement __repr__() but not ToString()!"; + } + } + + public class Bar + { + public override string ToString() + { + return "I implement ToString() but not __repr__()!"; + } + } + + public class BazBase + { + public override string ToString() + { + return "Base class implementing ToString()!"; + } + } + + public class BazMiddle : BazBase + { + public override string ToString() + { + return "Middle class implementing ToString()!"; + } + } + + //implements ToString via BazMiddle + public class Baz : BazMiddle + { + + } + + public class Quux + { + public string ToString(string format) + { + return "I implement ToString() with an argument!"; + } + } + + public class QuuzBase + { + protected string __repr__() + { + return "I implement __repr__ but it isn't public!"; + } + } + + public class Quuz : QuuzBase + { + + } + + public class Corge + { + public string __repr__(int i) + { + return "__repr__ implemention with input parameter!"; + } + } + + public class Grault + { + public int __repr__() + { + return "__repr__ implemention with wrong return type!".Length; + } + } + } +} diff --git a/src/testing/conversiontest.cs b/src/testing/conversiontest.cs index 06ab7cb4e..1f9d64e1b 100644 --- a/src/testing/conversiontest.cs +++ b/src/testing/conversiontest.cs @@ -1,7 +1,9 @@ namespace Python.Test { + using System.Collections.Generic; + /// - /// Supports units tests for field access. + /// Supports unit tests for field access. /// public class ConversionTest { @@ -32,6 +34,7 @@ public ConversionTest() public byte[] ByteArrayField; public sbyte[] SByteArrayField; + public readonly List ListField = new List(); public T? Echo(T? arg) where T: struct { return arg; diff --git a/src/testing/methodtest.cs b/src/testing/methodtest.cs index cf653f9f9..91836b727 100644 --- a/src/testing/methodtest.cs +++ b/src/testing/methodtest.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Runtime.InteropServices; namespace Python.Test { @@ -651,6 +652,38 @@ public static string Casesensitive() { return "Casesensitive"; } + + public static string DefaultParams(int a=0, int b=0, int c=0, int d=0) + { + return string.Format("{0}{1}{2}{3}", a, b, c, d); + } + + public static string OptionalParams([Optional]int a, [Optional]int b, [Optional]int c, [Optional] int d) + { + return string.Format("{0}{1}{2}{3}", a, b, c, d); + } + + public static bool OptionalParams_TestMissing([Optional]object a) + { + return a == Type.Missing; + } + + public static bool OptionalParams_TestReferenceType([Optional]string a) + { + return a == null; + } + + public static string OptionalAndDefaultParams([Optional]int a, [Optional]int b, int c=0, int d=0) + { + return string.Format("{0}{1}{2}{3}", a, b, c, d); + } + + public static string OptionalAndDefaultParams2([Optional]int a, [Optional]int b, [Optional, DefaultParameterValue(1)]int c, int d = 2) + { + return string.Format("{0}{1}{2}{3}", a, b, c, d); + } + + } diff --git a/src/testing/mp_lengthtest.cs b/src/testing/mp_lengthtest.cs new file mode 100644 index 000000000..a4f3e8c25 --- /dev/null +++ b/src/testing/mp_lengthtest.cs @@ -0,0 +1,171 @@ +using System; +using System.Collections; +using System.Collections.Generic; + +namespace Python.Test +{ + public class MpLengthCollectionTest : ICollection + { + private readonly List items; + + public MpLengthCollectionTest() + { + SyncRoot = new object(); + items = new List + { + 1, + 2, + 3 + }; + } + + public int Count => items.Count; + + public object SyncRoot { get; private set; } + + public bool IsSynchronized => false; + + public void CopyTo(Array array, int index) + { + throw new NotImplementedException(); + } + + public IEnumerator GetEnumerator() + { + throw new NotImplementedException(); + } + } + + public class MpLengthExplicitCollectionTest : ICollection + { + private readonly List items; + private readonly object syncRoot; + + public MpLengthExplicitCollectionTest() + { + syncRoot = new object(); + items = new List + { + 9, + 10 + }; + } + int ICollection.Count => items.Count; + + object ICollection.SyncRoot => syncRoot; + + bool ICollection.IsSynchronized => false; + + void ICollection.CopyTo(Array array, int index) + { + throw new NotImplementedException(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + throw new NotImplementedException(); + } + } + + public class MpLengthGenericCollectionTest : ICollection + { + private readonly List items; + + public MpLengthGenericCollectionTest() { + SyncRoot = new object(); + items = new List(); + } + + public int Count => items.Count; + + public object SyncRoot { get; private set; } + + public bool IsSynchronized => false; + + public bool IsReadOnly => false; + + public void Add(T item) + { + items.Add(item); + } + + public void Clear() + { + items.Clear(); + } + + public bool Contains(T item) + { + return items.Contains(item); + } + + public void CopyTo(T[] array, int arrayIndex) + { + items.CopyTo(array, arrayIndex); + } + + public IEnumerator GetEnumerator() + { + return ((IEnumerable)items).GetEnumerator(); + } + + public bool Remove(T item) + { + return items.Remove(item); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return items.GetEnumerator(); + } + } + + public class MpLengthExplicitGenericCollectionTest : ICollection + { + private readonly List items; + + public MpLengthExplicitGenericCollectionTest() + { + items = new List(); + } + + int ICollection.Count => items.Count; + + bool ICollection.IsReadOnly => false; + + public void Add(T item) + { + items.Add(item); + } + + void ICollection.Clear() + { + items.Clear(); + } + + bool ICollection.Contains(T item) + { + return items.Contains(item); + } + + void ICollection.CopyTo(T[] array, int arrayIndex) + { + items.CopyTo(array, arrayIndex); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return items.GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return ((IEnumerable)items).GetEnumerator(); + } + + bool ICollection.Remove(T item) + { + return items.Remove(item); + } + } +} diff --git a/src/testing/nonexportable.cs b/src/testing/nonexportable.cs new file mode 100644 index 000000000..a29c78589 --- /dev/null +++ b/src/testing/nonexportable.cs @@ -0,0 +1,8 @@ +namespace Python.Test +{ + using Python.Runtime; + + // this class should not be visible to Python + [PyExport(false)] + public class NonExportable { } +} diff --git a/src/tests/test_array.py b/src/tests/test_array.py index 7ccadddff..b492a66d3 100644 --- a/src/tests/test_array.py +++ b/src/tests/test_array.py @@ -1337,3 +1337,34 @@ def test_array_abuse(): with pytest.raises(TypeError): desc = Test.PublicArrayTest.__dict__['__setitem__'] desc(0, 0, 0) + + +@pytest.mark.skipif(PY2, reason="Only applies in Python 3") +def test_iterator_to_array(): + from System import Array, String + + d = {"a": 1, "b": 2, "c": 3} + keys_iterator = iter(d.keys()) + arr = Array[String](keys_iterator) + + Array.Sort(arr) + + assert arr[0] == "a" + assert arr[1] == "b" + assert arr[2] == "c" + + +@pytest.mark.skipif(PY2, reason="Only applies in Python 3") +def test_dict_keys_to_array(): + from System import Array, String + + d = {"a": 1, "b": 2, "c": 3} + d_keys = d.keys() + arr = Array[String](d_keys) + + Array.Sort(arr) + + assert arr[0] == "a" + assert arr[1] == "b" + assert arr[2] == "c" + diff --git a/src/tests/test_class.py b/src/tests/test_class.py index 612ce442e..6db3c36b7 100644 --- a/src/tests/test_class.py +++ b/src/tests/test_class.py @@ -14,7 +14,6 @@ def test_basic_reference_type(): """Test usage of CLR defined reference types.""" assert System.String.Empty == "" - def test_basic_value_type(): """Test usage of CLR defined value types.""" assert System.Int32.MaxValue == 2147483647 @@ -29,7 +28,6 @@ def test_class_standard_attrs(): assert isinstance(ClassTest.__dict__, DictProxyType) assert len(ClassTest.__doc__) > 0 - def test_class_docstrings(): """Test standard class docstring generation""" from Python.Test import ClassTest @@ -58,6 +56,14 @@ def test_non_public_class(): with pytest.raises(AttributeError): _ = Test.InternalClass +def test_non_exported(): + """Test [PyExport(false)]""" + with pytest.raises(ImportError): + from Python.Test import NonExportable + + with pytest.raises(AttributeError): + _ = Test.NonExportable + def test_basic_subclass(): """Test basic subclass of a managed class.""" diff --git a/src/tests/test_conversion.py b/src/tests/test_conversion.py index 0ba10a80e..1386a0358 100644 --- a/src/tests/test_conversion.py +++ b/src/tests/test_conversion.py @@ -6,6 +6,8 @@ import System import pytest from Python.Test import ConversionTest, UnicodeString +from Python.Runtime import PyObjectConversions +from Python.Runtime.Codecs import RawProxyEncoder from ._compat import indexbytes, long, unichr, text_type, PY2, PY3 @@ -595,11 +597,10 @@ def test_object_conversion(): # need to test subclass here - with pytest.raises(TypeError): - class Foo(object): - pass - ob = ConversionTest() - ob.ObjectField = Foo + class Foo(object): + pass + ob.ObjectField = Foo + assert ob.ObjectField == Foo def test_enum_conversion(): @@ -701,3 +702,19 @@ def test_sbyte_array_conversion(): array = ob.SByteArrayField for i, _ in enumerate(value): assert array[i] == indexbytes(value, i) + +def test_codecs(): + """Test codec registration from Python""" + class ListAsRawEncoder(RawProxyEncoder): + __namespace__ = "Python.Test" + def CanEncode(self, clr_type): + return clr_type.Name == "List`1" and clr_type.Namespace == "System.Collections.Generic" + + list_raw_encoder = ListAsRawEncoder() + PyObjectConversions.RegisterEncoder(list_raw_encoder) + + ob = ConversionTest() + + l = ob.ListField + l.Add(42) + assert ob.ListField.Count == 1 diff --git a/src/tests/test_exceptions.py b/src/tests/test_exceptions.py index a10d9a183..c2f18d443 100644 --- a/src/tests/test_exceptions.py +++ b/src/tests/test_exceptions.py @@ -282,11 +282,8 @@ def test_python_compat_of_managed_exceptions(): assert e.args == (msg,) assert isinstance(e.args, tuple) - if PY3: - strexp = "OverflowException('Simple message" - assert repr(e)[:len(strexp)] == strexp - elif PY2: - assert repr(e) == "OverflowException(u'Simple message',)" + strexp = "OverflowException('Simple message" + assert repr(e)[:len(strexp)] == strexp def test_exception_is_instance_of_system_object(): diff --git a/src/tests/test_indexer.py b/src/tests/test_indexer.py index 6f18550d9..ca4fd3b89 100644 --- a/src/tests/test_indexer.py +++ b/src/tests/test_indexer.py @@ -438,13 +438,13 @@ def test_object_indexer(): ob[long(1)] = "long" assert ob[long(1)] == "long" - with pytest.raises(TypeError): - class Eggs(object): - pass + class Eggs(object): + pass - key = Eggs() - ob = Test.ObjectIndexerTest() - ob[key] = "wrong" + key = Eggs() + ob = Test.ObjectIndexerTest() + ob[key] = "eggs_key" + assert ob[key] == "eggs_key" def test_interface_indexer(): diff --git a/src/tests/test_method.py b/src/tests/test_method.py index ad678611b..69f1b5e72 100644 --- a/src/tests/test_method.py +++ b/src/tests/test_method.py @@ -257,6 +257,37 @@ def test_non_params_array_in_last_place(): result = MethodTest.TestNonParamsArrayInLastPlace(1, 2, 3) assert result +def test_params_methods_with_no_params(): + """Tests that passing no arguments to a params method + passes an empty array""" + result = MethodTest.TestValueParamsArg() + assert len(result) == 0 + + result = MethodTest.TestOneArgWithParams('Some String') + assert len(result) == 0 + + result = MethodTest.TestTwoArgWithParams('Some String', 'Some Other String') + assert len(result) == 0 + +def test_params_methods_with_non_lists(): + """Tests that passing single parameters to a params + method will convert into an array on the .NET side""" + result = MethodTest.TestOneArgWithParams('Some String', [1, 2, 3, 4]) + assert len(result) == 4 + + result = MethodTest.TestOneArgWithParams('Some String', 1, 2, 3, 4) + assert len(result) == 4 + + result = MethodTest.TestOneArgWithParams('Some String', [5]) + assert len(result) == 1 + + result = MethodTest.TestOneArgWithParams('Some String', 5) + assert len(result) == 1 + +def test_params_method_with_lists(): + """Tests passing multiple lists to a params object[] method""" + result = MethodTest.TestObjectParamsArg([],[]) + assert len(result) == 2 def test_string_out_params(): """Test use of string out-parameters.""" @@ -776,6 +807,9 @@ def test_no_object_in_param(): res = MethodTest.TestOverloadedNoObject(5) assert res == "Got int" + + res = MethodTest.TestOverloadedNoObject(i=7) + assert res == "Got int" with pytest.raises(TypeError): MethodTest.TestOverloadedNoObject("test") @@ -787,9 +821,15 @@ def test_object_in_param(): res = MethodTest.TestOverloadedObject(5) assert res == "Got int" + + res = MethodTest.TestOverloadedObject(i=7) + assert res == "Got int" res = MethodTest.TestOverloadedObject("test") assert res == "Got object" + + res = MethodTest.TestOverloadedObject(o="test") + assert res == "Got object" def test_object_in_multiparam(): @@ -813,6 +853,42 @@ def test_object_in_multiparam(): res = MethodTest.TestOverloadedObjectTwo(7.24, 7.24) assert res == "Got object-object" + res = MethodTest.TestOverloadedObjectTwo(a=5, b=5) + assert res == "Got int-int" + + res = MethodTest.TestOverloadedObjectTwo(5, b=5) + assert res == "Got int-int" + + res = MethodTest.TestOverloadedObjectTwo(a=5, b="foo") + assert res == "Got int-string" + + res = MethodTest.TestOverloadedObjectTwo(5, b="foo") + assert res == "Got int-string" + + res = MethodTest.TestOverloadedObjectTwo(a="foo", b=7.24) + assert res == "Got string-object" + + res = MethodTest.TestOverloadedObjectTwo("foo", b=7.24) + assert res == "Got string-object" + + res = MethodTest.TestOverloadedObjectTwo(a="foo", b="bar") + assert res == "Got string-string" + + res = MethodTest.TestOverloadedObjectTwo("foo", b="bar") + assert res == "Got string-string" + + res = MethodTest.TestOverloadedObjectTwo(a="foo", b=5) + assert res == "Got string-int" + + res = MethodTest.TestOverloadedObjectTwo("foo", b=5) + assert res == "Got string-int" + + res = MethodTest.TestOverloadedObjectTwo(a=7.24, b=7.24) + assert res == "Got object-object" + + res = MethodTest.TestOverloadedObjectTwo(7.24, b=7.24) + assert res == "Got object-object" + def test_object_in_multiparam_exception(): """Test method with object multiparams behaves""" @@ -966,3 +1042,120 @@ def test_getting_overloaded_constructor_binding_does_not_leak_ref_count(): # simple test refCount = sys.getrefcount(PlainOldClass.Overloads[int]) assert refCount == 1 + + +def test_default_params(): + # all positional parameters + res = MethodTest.DefaultParams(1,2,3,4) + assert res == "1234" + + res = MethodTest.DefaultParams(1, 2, 3) + assert res == "1230" + + res = MethodTest.DefaultParams(1, 2) + assert res == "1200" + + res = MethodTest.DefaultParams(1) + assert res == "1000" + + res = MethodTest.DefaultParams(a=2) + assert res == "2000" + + res = MethodTest.DefaultParams(b=3) + assert res == "0300" + + res = MethodTest.DefaultParams(c=4) + assert res == "0040" + + res = MethodTest.DefaultParams(d=7) + assert res == "0007" + + res = MethodTest.DefaultParams(a=2, c=5) + assert res == "2050" + + res = MethodTest.DefaultParams(1, d=7, c=3) + assert res == "1037" + + with pytest.raises(TypeError): + MethodTest.DefaultParams(1,2,3,4,5) + +def test_optional_params(): + res = MethodTest.OptionalParams(1, 2, 3, 4) + assert res == "1234" + + res = MethodTest.OptionalParams(1, 2, 3) + assert res == "1230" + + res = MethodTest.OptionalParams(1, 2) + assert res == "1200" + + res = MethodTest.OptionalParams(1) + assert res == "1000" + + res = MethodTest.OptionalParams(a=2) + assert res == "2000" + + res = MethodTest.OptionalParams(b=3) + assert res == "0300" + + res = MethodTest.OptionalParams(c=4) + assert res == "0040" + + res = MethodTest.OptionalParams(d=7) + assert res == "0007" + + res = MethodTest.OptionalParams(a=2, c=5) + assert res == "2050" + + res = MethodTest.OptionalParams(1, d=7, c=3) + assert res == "1037" + + res = MethodTest.OptionalParams_TestMissing() + assert res == True + + res = MethodTest.OptionalParams_TestMissing(None) + assert res == False + + res = MethodTest.OptionalParams_TestMissing(a = None) + assert res == False + + res = MethodTest.OptionalParams_TestMissing(a='hi') + assert res == False + + res = MethodTest.OptionalParams_TestReferenceType() + assert res == True + + res = MethodTest.OptionalParams_TestReferenceType(None) + assert res == True + + res = MethodTest.OptionalParams_TestReferenceType(a=None) + assert res == True + + res = MethodTest.OptionalParams_TestReferenceType('hi') + assert res == False + + res = MethodTest.OptionalParams_TestReferenceType(a='hi') + assert res == False + +def test_optional_and_default_params(): + + res = MethodTest.OptionalAndDefaultParams() + assert res == "0000" + + res = MethodTest.OptionalAndDefaultParams(1) + assert res == "1000" + + res = MethodTest.OptionalAndDefaultParams(1, c=4) + assert res == "1040" + + res = MethodTest.OptionalAndDefaultParams(b=4, c=7) + assert res == "0470" + + res = MethodTest.OptionalAndDefaultParams2() + assert res == "0012" + + res = MethodTest.OptionalAndDefaultParams2(a=1,b=2,c=3,d=4) + assert res == "1234" + + res = MethodTest.OptionalAndDefaultParams2(b=2, c=3) + assert res == "0232" diff --git a/src/tests/test_mp_length.py b/src/tests/test_mp_length.py new file mode 100644 index 000000000..c96ac77d1 --- /dev/null +++ b/src/tests/test_mp_length.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- + +"""Test __len__ for .NET classes implementing ICollection/ICollection.""" + +import System +import pytest +from Python.Test import MpLengthCollectionTest, MpLengthExplicitCollectionTest, MpLengthGenericCollectionTest, MpLengthExplicitGenericCollectionTest + +def test_simple___len__(): + """Test __len__ for simple ICollection implementers""" + import System + import System.Collections.Generic + l = System.Collections.Generic.List[int]() + assert len(l) == 0 + l.Add(5) + l.Add(6) + assert len(l) == 2 + + d = System.Collections.Generic.Dictionary[int, int]() + assert len(d) == 0 + d.Add(4, 5) + assert len(d) == 1 + + a = System.Array[int]([0,1,2,3]) + assert len(a) == 4 + +def test_custom_collection___len__(): + """Test __len__ for custom collection class""" + s = MpLengthCollectionTest() + assert len(s) == 3 + +def test_custom_collection_explicit___len__(): + """Test __len__ for custom collection class that explicitly implements ICollection""" + s = MpLengthExplicitCollectionTest() + assert len(s) == 2 + +def test_custom_generic_collection___len__(): + """Test __len__ for custom generic collection class""" + s = MpLengthGenericCollectionTest[int]() + s.Add(1) + s.Add(2) + assert len(s) == 2 + +def test_custom_generic_collection_explicit___len__(): + """Test __len__ for custom generic collection that explicity implements ICollection""" + s = MpLengthExplicitGenericCollectionTest[int]() + s.Add(1) + s.Add(10) + assert len(s) == 2 diff --git a/src/tests/test_repr.py b/src/tests/test_repr.py new file mode 100644 index 000000000..5131f5d88 --- /dev/null +++ b/src/tests/test_repr.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- + +"""Test __repr__ output""" + +import System +import pytest +from Python.Test import ReprTest + +def test_basic(): + """Test Point class which implements both ToString and __repr__ without inheritance""" + ob = ReprTest.Point(1,2) + # point implements ToString() and __repr__() + assert ob.__repr__() == "Point(1,2)" + assert str(ob) == "Python.Test.ReprTest+Point: X=1, Y=2" + +def test_system_string(): + """Test system string""" + ob = System.String("hello") + assert str(ob) == "hello" + assert " + diff --git a/tools/geninterop/geninterop.py b/tools/geninterop/geninterop.py index f8ef8e561..902296229 100644 --- a/tools/geninterop/geninterop.py +++ b/tools/geninterop/geninterop.py @@ -76,6 +76,8 @@ def visit(self, node): self.visit_struct(node) elif isinstance(node, c_ast.Decl): self.visit_decl(node) + elif isinstance(node, c_ast.FuncDecl): + self.visit_funcdecl(node) elif isinstance(node, c_ast.PtrDecl): self.visit_ptrdecl(node) elif isinstance(node, c_ast.IdentifierType): @@ -110,6 +112,9 @@ def visit_struct(self, struct): def visit_decl(self, decl): self.visit(decl.type) + def visit_funcdecl(self, funcdecl): + self.visit(funcdecl.type) + def visit_ptrdecl(self, ptrdecl): self.__ptr_decl_depth += 1 self.visit(ptrdecl.type) @@ -169,6 +174,8 @@ def preprocess_python_headers(): include_py = sysconfig.get_config_var("INCLUDEPY") include_dirs.append(include_py) + include_args = [c for p in include_dirs for c in ["-I", p]] + defines = [ "-D", "__attribute__(x)=", "-D", "__inline__=inline", @@ -177,6 +184,13 @@ def preprocess_python_headers(): "-D", "_POSIX_THREADS" ] + if os.name == 'nt': + defines.extend([ + "-D", "__inline=inline", + "-D", "__ptr32=", + "-D", "__declspec(x)=", + ]) + if hasattr(sys, "abiflags"): if "d" in sys.abiflags: defines.extend(("-D", "PYTHON_WITH_PYDEBUG")) @@ -186,7 +200,7 @@ def preprocess_python_headers(): defines.extend(("-D", "PYTHON_WITH_WIDE_UNICODE")) python_h = os.path.join(include_py, "Python.h") - cmd = ["clang", "-pthread", "-I"] + include_dirs + defines + ["-E", python_h] + cmd = ["clang", "-pthread"] + include_args + defines + ["-E", python_h] # normalize as the parser doesn't like windows line endings. lines = [] @@ -216,7 +230,7 @@ def gen_interop_code(members): defines_str = " && ".join(defines) class_definition = """ // Auto-generated by %s. -// DO NOT MODIFIY BY HAND. +// DO NOT MODIFY BY HAND. #if %s diff --git a/tools/vswhere/vswhere.exe b/tools/vswhere/vswhere.exe index 3eb2df009..582e82868 100644 Binary files a/tools/vswhere/vswhere.exe and b/tools/vswhere/vswhere.exe differ diff --git a/tox.ini b/tox.ini deleted file mode 100644 index 3c9be5c63..000000000 --- a/tox.ini +++ /dev/null @@ -1,27 +0,0 @@ -[tox] -skip_missing_interpreters=True -envlist = - py27 - py33 - py34 - py35 - py36 - py37 - check - -[testenv] -deps = - pytest -commands = - {posargs:py.test} - -[testenv:check] -ignore_errors=True -skip_install=True -basepython=python3.5 -deps = - check-manifest - flake8 -commands = - flake8 src setup.py - python setup.py check --strict --metadata