Skip to content

Report AppVeyor build timings #1051

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Feb 21, 2020
Merged

Report AppVeyor build timings #1051

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9e3ed3a
report AppVeyor build timings
lostmsu Feb 20, 2020
103fa09
reduced number of iterations in performance tests to fit into AppVeyo…
lostmsu Feb 20, 2020
c4bbc8c
make timings output more prominent
lostmsu Feb 20, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion ci/appveyor_build_recipe.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
# Build `conda.recipe` only if this is a Pull_Request. Saves time for CI.

$stopwatch = [Diagnostics.Stopwatch]::StartNew()

$env:CONDA_PY = "$env:PY_VER"
# Use pre-installed miniconda. Note that location differs if 64bit
$env:CONDA_BLD = "C:\miniconda36"
Expand Down Expand Up @@ -30,7 +32,9 @@ if ($env:APPVEYOR_PULL_REQUEST_NUMBER -or $env:APPVEYOR_REPO_TAG_NAME -or $env:F

$CONDA_PKG=(conda build conda.recipe --output)
Copy-Item $CONDA_PKG .\dist\
Write-Host "Completed conda build recipe" -ForegroundColor "Green"

$timeSpent = $stopwatch.Elapsed
Write-Host "Completed conda build recipe in " $timeSpent -ForegroundColor "Green"

# Restore PATH back to original
$env:path = $old_path
Expand Down
28 changes: 28 additions & 0 deletions ci/appveyor_run_tests.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
# Script to simplify AppVeyor configuration and resolve path to tools

$stopwatch = [Diagnostics.Stopwatch]::StartNew()
[array]$timings = @()

# Test Runner framework being used for embedded tests
$CS_RUNNER = "nunit3-console"

Expand All @@ -25,6 +28,17 @@ $PY = Get-Command python
$CS_TESTS = ".\src\embed_tests\bin\Python.EmbeddingTest.dll"
$RUNTIME_DIR = ".\src\runtime\bin\"

function ReportTime {
param([string] $action)

$timeSpent = $stopwatch.Elapsed
$timings += [pscustomobject]@{action=$action; timeSpent=$timeSpent}
Write-Host $action " in " $timeSpent -ForegroundColor "Green"
$stopwatch.Restart()
}

ReportTime "Preparation done"

# Run python tests with C# coverage
Write-Host ("Starting Python tests") -ForegroundColor "Green"
.$OPENCOVER -register:user -searchdirs:"$RUNTIME_DIR" -output:py.coverage `
Expand All @@ -33,6 +47,9 @@ Write-Host ("Starting Python tests") -ForegroundColor "Green"
$PYTHON_STATUS = $LastExitCode
if ($PYTHON_STATUS -ne 0) {
Write-Host "Python tests failed, continuing to embedded tests" -ForegroundColor "Red"
ReportTime ""
} else {
ReportTime "Python tests completed"
}

# Run Embedded tests with C# coverage
Expand All @@ -44,7 +61,10 @@ Write-Host ("Starting embedded tests") -ForegroundColor "Green"
$CS_STATUS = $LastExitCode
if ($CS_STATUS -ne 0) {
Write-Host "Embedded tests failed" -ForegroundColor "Red"
ReportTime ""
} else {
ReportTime "Embedded tests completed"

# NuGet for pythonnet-2.3 only has 64-bit binary for Python 3.5
# the test is only built using modern stack
if (($env:PLATFORM -eq "x64") -and ($XPLAT) -and ($env:PYTHON_VERSION -eq "3.5")) {
Expand All @@ -60,6 +80,9 @@ if ($CS_STATUS -ne 0) {
$CS_PERF_STATUS = $LastExitCode
if ($CS_PERF_STATUS -ne 0) {
Write-Host "Performance tests (C#) failed" -ForegroundColor "Red"
ReportTime ""
} else {
ReportTime "Performance tests (C#) completed"
}
} else {
Write-Host ("Skipping performance tests for ", $env:PYTHON_VERSION) -ForegroundColor "Yellow"
Expand All @@ -82,9 +105,14 @@ if ($XPLAT){
$CS_STATUS = $LastExitCode
if ($CS_STATUS -ne 0) {
Write-Host "Embedded tests for netcoreapp2.0 failed" -ForegroundColor "Red"
ReportTime ""
} else {
ReportTime ".NET Core 2.0 tests completed"
}
}

Write-Host "Timings:" ($timings | Format-Table | Out-String) -ForegroundColor "Green"

# Set exit code to fail if either Python or Embedded tests failed
if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0 -or $CS_PERF_STATUS -ne 0) {
Write-Host "Tests failed" -ForegroundColor "Red"
Expand Down
4 changes: 2 additions & 2 deletions src/perf_tests/PythonCallingNetBenchmark.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ public void ReadInt64Property()
locals.SetItem("a", new NetObject().ToPython());
PythonEngine.Exec($@"
s = 0
for i in range(300000):
for i in range(50000):
s += a.{nameof(NetObject.LongProperty)}
", locals: locals.Handle);
}
Expand All @@ -32,7 +32,7 @@ public void WriteInt64Property() {
locals.SetItem("a", new NetObject().ToPython());
PythonEngine.Exec($@"
s = 0
for i in range(300000):
for i in range(50000):
a.{nameof(NetObject.LongProperty)} += i
", locals: locals.Handle);
}
Expand Down