Skip to content

CI for performance tests #992

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 25 commits into from
Feb 13, 2020
Merged

CI for performance tests #992

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
c9505df
attempted to add performance tests to CI
lostmsu Oct 22, 2019
949b55f
attempt to fix PerformanceTests xplat CI build
lostmsu Nov 21, 2019
1c92d0c
enabling building PerformanceTests for Mono
lostmsu Nov 21, 2019
4deda8b
use Ubuntu 18.04 Bionic Beaver image in Travis CI
lostmsu Nov 21, 2019
a7b9417
fixed AppVeyor path to Python.PerformanceTests.dll
lostmsu Nov 21, 2019
400ea33
fixed Mono deb sources to bionic
lostmsu Nov 21, 2019
01ea645
slightly relaxed perf target for WriteInt64Property
lostmsu Nov 21, 2019
419bddf
PerformanceTests: explicitly specify platform
lostmsu Nov 21, 2019
f36318f
use framework-specific build of perf tests in xplat and generic other…
lostmsu Nov 21, 2019
265b363
added reference to net461 reference assemblies nuget to perf tests
lostmsu Nov 21, 2019
e17f64e
added perf tests run to Travis CI
lostmsu Nov 21, 2019
433c016
fixed perf tests being build for Any CPU instead of x86
lostmsu Nov 21, 2019
74ba9d3
only run perf tests in Travis on Python 3.5
lostmsu Nov 21, 2019
54f9698
added performance tests to the old solution
lostmsu Nov 21, 2019
e2f192f
better error message for a failure to run benchmarks
lostmsu Nov 21, 2019
f69baf5
Revert "added performance tests to the old solution"
lostmsu Nov 21, 2019
618b841
appveyor: don't run perf tests in unsupported configurations
lostmsu Nov 21, 2019
123a701
commented out perf tests from Travis, as they are Windows-only due to…
lostmsu Nov 21, 2019
1993598
missing brace in appveyor_run_tests.ps1
lostmsu Nov 21, 2019
c46701d
fixed performance test Python version condition in AppVeyor
lostmsu Nov 21, 2019
340a6d1
explicitly notify when performance tests are skipped in AppVeyor
lostmsu Nov 21, 2019
a6c55d4
relax performance targets to ~10%, improve perf failure message
lostmsu Dec 19, 2019
171643c
Merge branch 'master' into features/PerfTests-CI
lostmsu Feb 12, 2020
031e8bb
travis: revert to xenial, as in bionic some tests crash
lostmsu Feb 12, 2020
0af4937
switch to the release of Microsoft.NETFramework.ReferenceAssemblies p…
lostmsu Feb 13, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ python:

env:
matrix:
- BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/
- BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH=""
- BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/ PERF_TESTS_PATH=net461/
- BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH="" PERF_TESTS_PATH=""

global:
- LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
Expand Down Expand Up @@ -45,6 +45,8 @@ install:
script:
- python -m pytest
- $RUN_TESTS src/embed_tests/bin/$EMBED_TESTS_PATH/Python.EmbeddingTest.dll
# does not work on Linux, because NuGet package for 2.3 is Windows only
# - "if [[ $TRAVIS_PYTHON_VERSION == '3.5' && $PERF_TESTS_PATH != '' ]]; then mono $NUNIT_PATH src/perf_tests/bin/$PERF_TESTS_PATH/Python.PerformanceTests.dll; fi"

after_script:
# Waiting on mono-coverage, SharpCover or xr.Baboon
Expand Down
32 changes: 28 additions & 4 deletions ci/appveyor_run_tests.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
# Test Runner framework being used for embedded tests
$CS_RUNNER = "nunit3-console"

$XPLAT = $env:BUILD_OPTS -eq "--xplat"

# Needed for ARCH specific runners(NUnit2/XUnit3). Skip for NUnit3
if ($FALSE -and $env:PLATFORM -eq "x86"){
$CS_RUNNER = $CS_RUNNER + "-x86"
Expand All @@ -11,7 +13,7 @@ if ($FALSE -and $env:PLATFORM -eq "x86"){
# Executable paths for OpenCover
# Note if OpenCover fails, it won't affect the exit codes.
$OPENCOVER = Resolve-Path .\packages\OpenCover.*\tools\OpenCover.Console.exe
if ($env:BUILD_OPTS -eq "--xplat"){
if ($XPLAT){
$CS_RUNNER = Resolve-Path $env:USERPROFILE\.nuget\packages\nunit.consolerunner\*\tools\"$CS_RUNNER".exe
}
else{
Expand Down Expand Up @@ -42,9 +44,31 @@ Write-Host ("Starting embedded tests") -ForegroundColor "Green"
$CS_STATUS = $LastExitCode
if ($CS_STATUS -ne 0) {
Write-Host "Embedded tests failed" -ForegroundColor "Red"
} else {
# NuGet for pythonnet-2.3 only has 64-bit binary for Python 3.5
# the test is only built using modern stack
if (($env:PLATFORM -eq "x64") -and ($XPLAT) -and ($env:PYTHON_VERSION -eq "3.5")) {
# Run C# Performance tests
Write-Host ("Starting performance tests") -ForegroundColor "Green"
if ($XPLAT) {
$CS_PERF_TESTS = ".\src\perf_tests\bin\net461\Python.PerformanceTests.dll"
}
else {
$CS_PERF_TESTS = ".\src\perf_tests\bin\Python.PerformanceTests.dll"
}
&"$CS_RUNNER" "$CS_PERF_TESTS"
$CS_PERF_STATUS = $LastExitCode
if ($CS_PERF_STATUS -ne 0) {
Write-Host "Performance tests (C#) failed" -ForegroundColor "Red"
}
} else {
Write-Host ("Skipping performance tests for ", $env:PYTHON_VERSION) -ForegroundColor "Yellow"
Write-Host ("on platform ", $env:PLATFORM, " xplat: ", $XPLAT) -ForegroundColor "Yellow"
$CS_PERF_STATUS = 0
}
}

if ($env:BUILD_OPTS -eq "--xplat"){
if ($XPLAT){
if ($env:PLATFORM -eq "x64") {
$DOTNET_CMD = "dotnet"
}
Expand All @@ -54,15 +78,15 @@ if ($env:BUILD_OPTS -eq "--xplat"){

# Run Embedded tests for netcoreapp2.0 (OpenCover currently does not supports dotnet core)
Write-Host ("Starting embedded tests for netcoreapp2.0") -ForegroundColor "Green"
&$DOTNET_CMD .\src\embed_tests\bin\netcoreapp2.0_publish\Python.EmbeddingTest.dll
&$DOTNET_CMD ".\src\embed_tests\bin\netcoreapp2.0_publish\Python.EmbeddingTest.dll"
$CS_STATUS = $LastExitCode
if ($CS_STATUS -ne 0) {
Write-Host "Embedded tests for netcoreapp2.0 failed" -ForegroundColor "Red"
}
}

# Set exit code to fail if either Python or Embedded tests failed
if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0) {
if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0 -or $CS_PERF_STATUS -ne 0) {
Write-Host "Tests failed" -ForegroundColor "Red"
$host.SetShouldExit(1)
}
120 changes: 72 additions & 48 deletions pythonnet.15.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,26 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repo", "Repo", "{441A0123-F
.editorconfig = .editorconfig
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CI", "CI", "{D301657F-5EAF-4534-B280-B858D651B2E5}"
ProjectSection(SolutionItems) = preProject
.travis.yml = .travis.yml
appveyor.yml = appveyor.yml
ci\appveyor_build_recipe.ps1 = ci\appveyor_build_recipe.ps1
ci\appveyor_run_tests.ps1 = ci\appveyor_run_tests.ps1
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{57F5D701-F265-4736-A5A2-07249E7A4DA3}"
ProjectSection(SolutionItems) = preProject
setup.py = setup.py
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "conda.recipe", "conda.recipe", "{7FD2404D-0CE8-4645-8DFB-766470E2150E}"
ProjectSection(SolutionItems) = preProject
conda.recipe\bld.bat = conda.recipe\bld.bat
conda.recipe\meta.yaml = conda.recipe\meta.yaml
conda.recipe\README.md = conda.recipe\README.md
EndProjectSection
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -313,54 +333,58 @@ Global
{F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64
{F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86
{F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.ActiveCfg = DebugMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.Build.0 = DebugMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.ActiveCfg = DebugMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.Build.0 = DebugMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.ActiveCfg = DebugMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.Build.0 = DebugMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|Any CPU.ActiveCfg = DebugMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x64.ActiveCfg = DebugMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x86.ActiveCfg = DebugMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|Any CPU.ActiveCfg = DebugWin|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|Any CPU.Build.0 = DebugWin|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.ActiveCfg = DebugWin|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.Build.0 = DebugWin|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.ActiveCfg = DebugWin|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.Build.0 = DebugWin|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|Any CPU.Build.0 = DebugWinPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.Build.0 = DebugWinPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.Build.0 = DebugWinPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.ActiveCfg = ReleaseMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.Build.0 = ReleaseMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.ActiveCfg = ReleaseMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.Build.0 = ReleaseMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.ActiveCfg = ReleaseMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.Build.0 = ReleaseMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x64.ActiveCfg = ReleaseMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x86.ActiveCfg = ReleaseMono|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|Any CPU.Build.0 = ReleaseWin|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.ActiveCfg = ReleaseWin|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.Build.0 = ReleaseWin|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.ActiveCfg = ReleaseWin|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.Build.0 = ReleaseWin|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|Any CPU.Build.0 = ReleaseWinPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|Any CPU
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.ActiveCfg = ReleaseWin|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.Build.0 = ReleaseWin|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.ActiveCfg = DebugWinPY3|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.Build.0 = DebugWinPY3|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.ActiveCfg = DebugWinPY3|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.Build.0 = DebugWinPY3|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x64.ActiveCfg = DebugMono|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x64.Build.0 = DebugMono|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x86.ActiveCfg = DebugMono|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x86.Build.0 = DebugMono|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.ActiveCfg = DebugWin|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.Build.0 = DebugWin|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.ActiveCfg = DebugWin|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.Build.0 = DebugWin|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.ActiveCfg = ReleaseWin|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.Build.0 = ReleaseWin|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.ActiveCfg = ReleaseWinPY3|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.Build.0 = ReleaseWinPY3|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.ActiveCfg = ReleaseWinPY3|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.Build.0 = ReleaseWinPY3|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x64.Build.0 = ReleaseMono|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x86.Build.0 = ReleaseMono|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.Build.0 = ReleaseWin|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.Build.0 = ReleaseWin|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down
10 changes: 10 additions & 0 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,16 @@ def build_extension(self, ext):
),
shell=use_shell,
)
subprocess.check_call(
" ".join(
cmd
+ [
'"/t:Python_PerformanceTests:publish"',
"/p:TargetFramework=net461",
]
),
shell=use_shell,
)
if DEVTOOLS == "Mono" or DEVTOOLS == "dotnet":
self._build_monoclr()

Expand Down
17 changes: 14 additions & 3 deletions src/perf_tests/BenchmarkTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,21 +21,23 @@ public void SetUp()
Environment.CurrentDirectory = Path.Combine(DeploymentRoot, "new");
this.summary = BenchmarkRunner.Run<PythonCallingNetBenchmark>();
Assert.IsNotEmpty(this.summary.Reports);
Assert.IsTrue(this.summary.Reports.All(r => r.Success));
Assert.IsTrue(
condition: this.summary.Reports.All(r => r.Success),
message: "BenchmarkDotNet failed to execute or collect results of performance tests. See logs above.");
}

[Test]
public void ReadInt64Property()
{
double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports);
Assert.LessOrEqual(optimisticPerfRatio, 0.68);
AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.66);
}

[Test]
public void WriteInt64Property()
{
double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports);
Assert.LessOrEqual(optimisticPerfRatio, 0.66);
AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.64);
}

static double GetOptimisticPerfRatio(
Expand All @@ -59,5 +61,14 @@ static double GetOptimisticPerfRatio(
}

public static string DeploymentRoot => Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);

public static void AssertPerformanceIsBetterOrSame(
double actual, double target,
double wiggleRoom = 1.1, [CallerMemberName] string testName = null) {
double threshold = target * wiggleRoom;
Assert.LessOrEqual(actual, threshold,
$"{testName}: {actual:F3} > {threshold:F3} (target: {target:F3})"
+ ": perf result is higher than the failure threshold.");
}
}
}
9 changes: 8 additions & 1 deletion src/perf_tests/Python.PerformanceTests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,21 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net461</TargetFramework>
<Configurations>DebugMono;DebugMonoPY3;ReleaseMono;ReleaseMonoPY3;DebugWin;DebugWinPY3;ReleaseWin;ReleaseWinPY3</Configurations>
<OutputPath>bin\</OutputPath>

<IsPackable>false</IsPackable>

<Platforms>x64;x86</Platforms>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.11.5" />
<PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.0">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="nunit" Version="3.12.0" />
<PackageReference Include="NUnit3TestAdapter" Version="3.13.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.2.0" />
Expand Down