Skip to content

Commit f5548e3

Browse files
lostmsulostmsu
authored
CI for performance tests (#992)
* attempted to add performance tests to CI * attempt to fix PerformanceTests xplat CI build * enabling building PerformanceTests for Mono * fixed AppVeyor path to Python.PerformanceTests.dll * fixed Mono deb sources to bionic * slightly relaxed perf target for WriteInt64Property * PerformanceTests: explicitly specify platform * use framework-specific build of perf tests in xplat and generic otherwise * added perf tests run to Travis CI * better error message for a failure to run benchmarks * appveyor: don't run perf tests in unsupported configurations * fixed performance test Python version condition in AppVeyor * explicitly notify when performance tests are skipped in AppVeyor * relax performance targets to ~10%, improve perf failure message * switch to the release of Microsoft.NETFramework.ReferenceAssemblies package
1 parent dbb88bc commit f5548e3

File tree

6 files changed

+136
-58
lines changed

6 files changed

+136
-58
lines changed

‎.travis.yml

+4-2
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ python:
99

1010
env:
1111
matrix:
12-
- BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/
13-
- BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH=""
12+
- BUILD_OPTS=--xplat NUNIT_PATH="~/.nuget/packages/nunit.consolerunner/3.*/tools/nunit3-console.exe" RUN_TESTS=dotnet EMBED_TESTS_PATH=netcoreapp2.0_publish/ PERF_TESTS_PATH=net461/
13+
- BUILD_OPTS="" NUNIT_PATH="./packages/NUnit.*/tools/nunit3-console.exe" RUN_TESTS="mono $NUNIT_PATH" EMBED_TESTS_PATH="" PERF_TESTS_PATH=""
1414

1515
global:
1616
- LD_PRELOAD=/lib/x86_64-linux-gnu/libSegFault.so
@@ -45,6 +45,8 @@ install:
4545
script:
4646
- python -m pytest
4747
- $RUN_TESTS src/embed_tests/bin/$EMBED_TESTS_PATH/Python.EmbeddingTest.dll
48+
# does not work on Linux, because NuGet package for 2.3 is Windows only
49+
# - "if [[ $TRAVIS_PYTHON_VERSION == '3.5' && $PERF_TESTS_PATH != '' ]]; then mono $NUNIT_PATH src/perf_tests/bin/$PERF_TESTS_PATH/Python.PerformanceTests.dll; fi"
4850

4951
after_script:
5052
# Waiting on mono-coverage, SharpCover or xr.Baboon

‎ci/appveyor_run_tests.ps1

+28-4
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
# Test Runner framework being used for embedded tests
44
$CS_RUNNER = "nunit3-console"
55

6+
$XPLAT = $env:BUILD_OPTS -eq "--xplat"
7+
68
# Needed for ARCH specific runners(NUnit2/XUnit3). Skip for NUnit3
79
if ($FALSE -and $env:PLATFORM -eq "x86"){
810
$CS_RUNNER = $CS_RUNNER + "-x86"
@@ -11,7 +13,7 @@ if ($FALSE -and $env:PLATFORM -eq "x86"){
1113
# Executable paths for OpenCover
1214
# Note if OpenCover fails, it won't affect the exit codes.
1315
$OPENCOVER = Resolve-Path .\packages\OpenCover.*\tools\OpenCover.Console.exe
14-
if ($env:BUILD_OPTS -eq "--xplat"){
16+
if ($XPLAT){
1517
$CS_RUNNER = Resolve-Path $env:USERPROFILE\.nuget\packages\nunit.consolerunner\*\tools\"$CS_RUNNER".exe
1618
}
1719
else{
@@ -42,9 +44,31 @@ Write-Host ("Starting embedded tests") -ForegroundColor "Green"
4244
$CS_STATUS = $LastExitCode
4345
if ($CS_STATUS -ne 0) {
4446
Write-Host "Embedded tests failed" -ForegroundColor "Red"
47+
} else {
48+
# NuGet for pythonnet-2.3 only has 64-bit binary for Python 3.5
49+
# the test is only built using modern stack
50+
if (($env:PLATFORM -eq "x64") -and ($XPLAT) -and ($env:PYTHON_VERSION -eq "3.5")) {
51+
# Run C# Performance tests
52+
Write-Host ("Starting performance tests") -ForegroundColor "Green"
53+
if ($XPLAT) {
54+
$CS_PERF_TESTS = ".\src\perf_tests\bin\net461\Python.PerformanceTests.dll"
55+
}
56+
else {
57+
$CS_PERF_TESTS = ".\src\perf_tests\bin\Python.PerformanceTests.dll"
58+
}
59+
&"$CS_RUNNER" "$CS_PERF_TESTS"
60+
$CS_PERF_STATUS = $LastExitCode
61+
if ($CS_PERF_STATUS -ne 0) {
62+
Write-Host "Performance tests (C#) failed" -ForegroundColor "Red"
63+
}
64+
} else {
65+
Write-Host ("Skipping performance tests for ", $env:PYTHON_VERSION) -ForegroundColor "Yellow"
66+
Write-Host ("on platform ", $env:PLATFORM, " xplat: ", $XPLAT) -ForegroundColor "Yellow"
67+
$CS_PERF_STATUS = 0
68+
}
4569
}
4670

47-
if ($env:BUILD_OPTS -eq "--xplat"){
71+
if ($XPLAT){
4872
if ($env:PLATFORM -eq "x64") {
4973
$DOTNET_CMD = "dotnet"
5074
}
@@ -54,15 +78,15 @@ if ($env:BUILD_OPTS -eq "--xplat"){
5478

5579
# Run Embedded tests for netcoreapp2.0 (OpenCover currently does not supports dotnet core)
5680
Write-Host ("Starting embedded tests for netcoreapp2.0") -ForegroundColor "Green"
57-
&$DOTNET_CMD .\src\embed_tests\bin\netcoreapp2.0_publish\Python.EmbeddingTest.dll
81+
&$DOTNET_CMD ".\src\embed_tests\bin\netcoreapp2.0_publish\Python.EmbeddingTest.dll"
5882
$CS_STATUS = $LastExitCode
5983
if ($CS_STATUS -ne 0) {
6084
Write-Host "Embedded tests for netcoreapp2.0 failed" -ForegroundColor "Red"
6185
}
6286
}
6387

6488
# Set exit code to fail if either Python or Embedded tests failed
65-
if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0) {
89+
if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0 -or $CS_PERF_STATUS -ne 0) {
6690
Write-Host "Tests failed" -ForegroundColor "Red"
6791
$host.SetShouldExit(1)
6892
}

‎pythonnet.15.sln

+72-48
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,26 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Repo", "Repo", "{441A0123-F
1919
.editorconfig = .editorconfig
2020
EndProjectSection
2121
EndProject
22+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CI", "CI", "{D301657F-5EAF-4534-B280-B858D651B2E5}"
23+
ProjectSection(SolutionItems) = preProject
24+
.travis.yml = .travis.yml
25+
appveyor.yml = appveyor.yml
26+
ci\appveyor_build_recipe.ps1 = ci\appveyor_build_recipe.ps1
27+
ci\appveyor_run_tests.ps1 = ci\appveyor_run_tests.ps1
28+
EndProjectSection
29+
EndProject
30+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{57F5D701-F265-4736-A5A2-07249E7A4DA3}"
31+
ProjectSection(SolutionItems) = preProject
32+
setup.py = setup.py
33+
EndProjectSection
34+
EndProject
35+
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "conda.recipe", "conda.recipe", "{7FD2404D-0CE8-4645-8DFB-766470E2150E}"
36+
ProjectSection(SolutionItems) = preProject
37+
conda.recipe\bld.bat = conda.recipe\bld.bat
38+
conda.recipe\meta.yaml = conda.recipe\meta.yaml
39+
conda.recipe\README.md = conda.recipe\README.md
40+
EndProjectSection
41+
EndProject
2242
Global
2343
GlobalSection(SolutionConfigurationPlatforms) = preSolution
2444
Debug|Any CPU = Debug|Any CPU
@@ -313,54 +333,58 @@ Global
313333
{F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64
314334
{F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86
315335
{F94B547A-E97E-4500-8D53-B4D64D076E5F}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86
316-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.ActiveCfg = DebugMono|Any CPU
317-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.Build.0 = DebugMono|Any CPU
318-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.ActiveCfg = DebugMono|Any CPU
319-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.Build.0 = DebugMono|Any CPU
320-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.ActiveCfg = DebugMono|Any CPU
321-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.Build.0 = DebugMono|Any CPU
322-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|Any CPU.ActiveCfg = DebugMono|Any CPU
323-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x64.ActiveCfg = DebugMono|Any CPU
324-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x86.ActiveCfg = DebugMono|Any CPU
325-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|Any CPU
326-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|Any CPU
327-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|Any CPU
328-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|Any CPU.ActiveCfg = DebugWin|Any CPU
329-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|Any CPU.Build.0 = DebugWin|Any CPU
330-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.ActiveCfg = DebugWin|Any CPU
331-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.Build.0 = DebugWin|Any CPU
332-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.ActiveCfg = DebugWin|Any CPU
333-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.Build.0 = DebugWin|Any CPU
334-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|Any CPU
335-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|Any CPU.Build.0 = DebugWinPY3|Any CPU
336-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|Any CPU
337-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.Build.0 = DebugWinPY3|Any CPU
338-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|Any CPU
339-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.Build.0 = DebugWinPY3|Any CPU
340-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.ActiveCfg = ReleaseMono|Any CPU
341-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.Build.0 = ReleaseMono|Any CPU
342-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.ActiveCfg = ReleaseMono|Any CPU
343-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.Build.0 = ReleaseMono|Any CPU
344-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.ActiveCfg = ReleaseMono|Any CPU
345-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.Build.0 = ReleaseMono|Any CPU
346-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|Any CPU
347-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x64.ActiveCfg = ReleaseMono|Any CPU
348-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x86.ActiveCfg = ReleaseMono|Any CPU
349-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|Any CPU
350-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|Any CPU
351-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|Any CPU
352-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|Any CPU
353-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|Any CPU.Build.0 = ReleaseWin|Any CPU
354-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.ActiveCfg = ReleaseWin|Any CPU
355-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.Build.0 = ReleaseWin|Any CPU
356-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.ActiveCfg = ReleaseWin|Any CPU
357-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.Build.0 = ReleaseWin|Any CPU
358-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|Any CPU
359-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|Any CPU.Build.0 = ReleaseWinPY3|Any CPU
360-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|Any CPU
361-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|Any CPU
362-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|Any CPU
363-
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|Any CPU
336+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.ActiveCfg = ReleaseWin|x86
337+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|Any CPU.Build.0 = ReleaseWin|x86
338+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.ActiveCfg = DebugWinPY3|x64
339+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x64.Build.0 = DebugWinPY3|x64
340+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.ActiveCfg = DebugWinPY3|x86
341+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Debug|x86.Build.0 = DebugWinPY3|x86
342+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|Any CPU.ActiveCfg = DebugMono|x86
343+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x64.ActiveCfg = DebugMono|x64
344+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x64.Build.0 = DebugMono|x64
345+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x86.ActiveCfg = DebugMono|x86
346+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMono|x86.Build.0 = DebugMono|x86
347+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|Any CPU.ActiveCfg = DebugMonoPY3|x86
348+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x64.ActiveCfg = DebugMonoPY3|x64
349+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x64.Build.0 = DebugMonoPY3|x64
350+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x86.ActiveCfg = DebugMonoPY3|x86
351+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugMonoPY3|x86.Build.0 = DebugMonoPY3|x86
352+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|Any CPU.ActiveCfg = DebugWin|x86
353+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.ActiveCfg = DebugWin|x64
354+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x64.Build.0 = DebugWin|x64
355+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.ActiveCfg = DebugWin|x86
356+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWin|x86.Build.0 = DebugWin|x86
357+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|Any CPU.ActiveCfg = DebugWinPY3|x86
358+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.ActiveCfg = DebugWinPY3|x64
359+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x64.Build.0 = DebugWinPY3|x64
360+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.ActiveCfg = DebugWinPY3|x86
361+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.DebugWinPY3|x86.Build.0 = DebugWinPY3|x86
362+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.ActiveCfg = ReleaseWin|x86
363+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|Any CPU.Build.0 = ReleaseWin|x86
364+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.ActiveCfg = ReleaseWinPY3|x64
365+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x64.Build.0 = ReleaseWinPY3|x64
366+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.ActiveCfg = ReleaseWinPY3|x86
367+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.Release|x86.Build.0 = ReleaseWinPY3|x86
368+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|Any CPU.ActiveCfg = ReleaseMono|x86
369+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x64.ActiveCfg = ReleaseMono|x64
370+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x64.Build.0 = ReleaseMono|x64
371+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x86.ActiveCfg = ReleaseMono|x86
372+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMono|x86.Build.0 = ReleaseMono|x86
373+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|Any CPU.ActiveCfg = ReleaseMonoPY3|x86
374+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x64.ActiveCfg = ReleaseMonoPY3|x64
375+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x64.Build.0 = ReleaseMonoPY3|x64
376+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x86.ActiveCfg = ReleaseMonoPY3|x86
377+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseMonoPY3|x86.Build.0 = ReleaseMonoPY3|x86
378+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|Any CPU.ActiveCfg = ReleaseWin|x86
379+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.ActiveCfg = ReleaseWin|x64
380+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x64.Build.0 = ReleaseWin|x64
381+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.ActiveCfg = ReleaseWin|x86
382+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWin|x86.Build.0 = ReleaseWin|x86
383+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|Any CPU.ActiveCfg = ReleaseWinPY3|x86
384+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.ActiveCfg = ReleaseWinPY3|x64
385+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x64.Build.0 = ReleaseWinPY3|x64
386+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.ActiveCfg = ReleaseWinPY3|x86
387+
{6FB0D091-9CEC-4DCC-8701-C40F9BFC9EDE}.ReleaseWinPY3|x86.Build.0 = ReleaseWinPY3|x86
364388
EndGlobalSection
365389
GlobalSection(SolutionProperties) = preSolution
366390
HideSolutionNode = FALSE

‎setup.py

+10
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,16 @@ def build_extension(self, ext):
362362
),
363363
shell=use_shell,
364364
)
365+
subprocess.check_call(
366+
" ".join(
367+
cmd
368+
+ [
369+
'"/t:Python_PerformanceTests:publish"',
370+
"/p:TargetFramework=net461",
371+
]
372+
),
373+
shell=use_shell,
374+
)
365375
if DEVTOOLS == "Mono" or DEVTOOLS == "dotnet":
366376
self._build_monoclr()
367377

‎src/perf_tests/BenchmarkTests.cs

+14-3
Original file line numberDiff line numberDiff line change
@@ -21,21 +21,23 @@ public void SetUp()
2121
Environment.CurrentDirectory = Path.Combine(DeploymentRoot, "new");
2222
this.summary = BenchmarkRunner.Run<PythonCallingNetBenchmark>();
2323
Assert.IsNotEmpty(this.summary.Reports);
24-
Assert.IsTrue(this.summary.Reports.All(r => r.Success));
24+
Assert.IsTrue(
25+
condition: this.summary.Reports.All(r => r.Success),
26+
message: "BenchmarkDotNet failed to execute or collect results of performance tests. See logs above.");
2527
}
2628

2729
[Test]
2830
public void ReadInt64Property()
2931
{
3032
double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports);
31-
Assert.LessOrEqual(optimisticPerfRatio, 0.68);
33+
AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.66);
3234
}
3335

3436
[Test]
3537
public void WriteInt64Property()
3638
{
3739
double optimisticPerfRatio = GetOptimisticPerfRatio(this.summary.Reports);
38-
Assert.LessOrEqual(optimisticPerfRatio, 0.66);
40+
AssertPerformanceIsBetterOrSame(optimisticPerfRatio, target: 0.64);
3941
}
4042

4143
static double GetOptimisticPerfRatio(
@@ -59,5 +61,14 @@ static double GetOptimisticPerfRatio(
5961
}
6062

6163
public static string DeploymentRoot => Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
64+
65+
public static void AssertPerformanceIsBetterOrSame(
66+
double actual, double target,
67+
double wiggleRoom = 1.1, [CallerMemberName] string testName = null) {
68+
double threshold = target * wiggleRoom;
69+
Assert.LessOrEqual(actual, threshold,
70+
$"{testName}: {actual:F3} > {threshold:F3} (target: {target:F3})"
71+
+ ": perf result is higher than the failure threshold.");
72+
}
6273
}
6374
}

‎src/perf_tests/Python.PerformanceTests.csproj

+8-1
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,21 @@
1-
<Project Sdk="Microsoft.NET.Sdk">
1+
<Project Sdk="Microsoft.NET.Sdk">
22

33
<PropertyGroup>
44
<TargetFramework>net461</TargetFramework>
55
<Configurations>DebugMono;DebugMonoPY3;ReleaseMono;ReleaseMonoPY3;DebugWin;DebugWinPY3;ReleaseWin;ReleaseWinPY3</Configurations>
6+
<OutputPath>bin\</OutputPath>
67

78
<IsPackable>false</IsPackable>
9+
10+
<Platforms>x64;x86</Platforms>
811
</PropertyGroup>
912

1013
<ItemGroup>
1114
<PackageReference Include="BenchmarkDotNet" Version="0.11.5" />
15+
<PackageReference Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.0">
16+
<PrivateAssets>all</PrivateAssets>
17+
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
18+
</PackageReference>
1219
<PackageReference Include="nunit" Version="3.12.0" />
1320
<PackageReference Include="NUnit3TestAdapter" Version="3.13.0" />
1421
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.2.0" />

0 commit comments

Comments
 (0)