From 9e3ed3ae69c54dc26e9e7a15c5c9026ccfa7dfce Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 20 Feb 2020 12:33:03 -0800 Subject: [PATCH 1/3] report AppVeyor build timings --- ci/appveyor_build_recipe.ps1 | 6 +++++- ci/appveyor_run_tests.ps1 | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/ci/appveyor_build_recipe.ps1 b/ci/appveyor_build_recipe.ps1 index 84e0bc7c6..08eae8d5d 100644 --- a/ci/appveyor_build_recipe.ps1 +++ b/ci/appveyor_build_recipe.ps1 @@ -1,5 +1,7 @@ # Build `conda.recipe` only if this is a Pull_Request. Saves time for CI. +$stopwatch = [Diagnostics.Stopwatch]::StartNew() + $env:CONDA_PY = "$env:PY_VER" # Use pre-installed miniconda. Note that location differs if 64bit $env:CONDA_BLD = "C:\miniconda36" @@ -30,7 +32,9 @@ if ($env:APPVEYOR_PULL_REQUEST_NUMBER -or $env:APPVEYOR_REPO_TAG_NAME -or $env:F $CONDA_PKG=(conda build conda.recipe --output) Copy-Item $CONDA_PKG .\dist\ - Write-Host "Completed conda build recipe" -ForegroundColor "Green" + + $timeSpent = $stopwatch.Elapsed + Write-Host "Completed conda build recipe in " $timeSpent -ForegroundColor "Green" # Restore PATH back to original $env:path = $old_path diff --git a/ci/appveyor_run_tests.ps1 b/ci/appveyor_run_tests.ps1 index f94cfb11e..7bd632b19 100644 --- a/ci/appveyor_run_tests.ps1 +++ b/ci/appveyor_run_tests.ps1 @@ -1,5 +1,8 @@ # Script to simplify AppVeyor configuration and resolve path to tools +$stopwatch = [Diagnostics.Stopwatch]::StartNew() +[array]$timings = @() + # Test Runner framework being used for embedded tests $CS_RUNNER = "nunit3-console" @@ -25,6 +28,17 @@ $PY = Get-Command python $CS_TESTS = ".\src\embed_tests\bin\Python.EmbeddingTest.dll" $RUNTIME_DIR = ".\src\runtime\bin\" +function ReportTime { + param([string] $action) + + $timeSpent = $stopwatch.Elapsed + $timings += [pscustomobject]@{action=$action; timeSpent=$timeSpent} + Write-Host $action " in " $timeSpent -ForegroundColor "Green" + $stopwatch.Restart() +} + +ReportTime "Preparation done" + # Run python tests with C# coverage Write-Host ("Starting Python tests") -ForegroundColor "Green" .$OPENCOVER -register:user -searchdirs:"$RUNTIME_DIR" -output:py.coverage ` @@ -33,6 +47,9 @@ Write-Host ("Starting Python tests") -ForegroundColor "Green" $PYTHON_STATUS = $LastExitCode if ($PYTHON_STATUS -ne 0) { Write-Host "Python tests failed, continuing to embedded tests" -ForegroundColor "Red" + ReportTime "" +} else { + ReportTime "Python tests completed" } # Run Embedded tests with C# coverage @@ -44,7 +61,10 @@ Write-Host ("Starting embedded tests") -ForegroundColor "Green" $CS_STATUS = $LastExitCode if ($CS_STATUS -ne 0) { Write-Host "Embedded tests failed" -ForegroundColor "Red" + ReportTime "" } else { + ReportTime "Embedded tests completed" + # NuGet for pythonnet-2.3 only has 64-bit binary for Python 3.5 # the test is only built using modern stack if (($env:PLATFORM -eq "x64") -and ($XPLAT) -and ($env:PYTHON_VERSION -eq "3.5")) { @@ -60,6 +80,9 @@ if ($CS_STATUS -ne 0) { $CS_PERF_STATUS = $LastExitCode if ($CS_PERF_STATUS -ne 0) { Write-Host "Performance tests (C#) failed" -ForegroundColor "Red" + ReportTime "" + } else { + ReportTime "Performance tests (C#) completed" } } else { Write-Host ("Skipping performance tests for ", $env:PYTHON_VERSION) -ForegroundColor "Yellow" @@ -82,9 +105,14 @@ if ($XPLAT){ $CS_STATUS = $LastExitCode if ($CS_STATUS -ne 0) { Write-Host "Embedded tests for netcoreapp2.0 failed" -ForegroundColor "Red" + ReportTime "" + } else { + ReportTime ".NET Core 2.0 tests completed" } } +Write-Host ($timings | Format-Table | Out-String) + # Set exit code to fail if either Python or Embedded tests failed if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0 -or $CS_PERF_STATUS -ne 0) { Write-Host "Tests failed" -ForegroundColor "Red" From 103fa099015786b291bd8b009f6516099385bc2d Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 20 Feb 2020 14:40:29 -0800 Subject: [PATCH 2/3] reduced number of iterations in performance tests to fit into AppVeyor time limit --- src/perf_tests/PythonCallingNetBenchmark.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/perf_tests/PythonCallingNetBenchmark.cs b/src/perf_tests/PythonCallingNetBenchmark.cs index 4e9461d2e..ef668a911 100644 --- a/src/perf_tests/PythonCallingNetBenchmark.cs +++ b/src/perf_tests/PythonCallingNetBenchmark.cs @@ -19,7 +19,7 @@ public void ReadInt64Property() locals.SetItem("a", new NetObject().ToPython()); PythonEngine.Exec($@" s = 0 -for i in range(300000): +for i in range(50000): s += a.{nameof(NetObject.LongProperty)} ", locals: locals.Handle); } @@ -32,7 +32,7 @@ public void WriteInt64Property() { locals.SetItem("a", new NetObject().ToPython()); PythonEngine.Exec($@" s = 0 -for i in range(300000): +for i in range(50000): a.{nameof(NetObject.LongProperty)} += i ", locals: locals.Handle); } From c4bbc8cc1e3683f4a8df5ad18dcfcfa6717b5314 Mon Sep 17 00:00:00 2001 From: Victor Milovanov Date: Thu, 20 Feb 2020 15:40:20 -0800 Subject: [PATCH 3/3] make timings output more prominent --- ci/appveyor_run_tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/appveyor_run_tests.ps1 b/ci/appveyor_run_tests.ps1 index 7bd632b19..cb1c68eed 100644 --- a/ci/appveyor_run_tests.ps1 +++ b/ci/appveyor_run_tests.ps1 @@ -111,7 +111,7 @@ if ($XPLAT){ } } -Write-Host ($timings | Format-Table | Out-String) +Write-Host "Timings:" ($timings | Format-Table | Out-String) -ForegroundColor "Green" # Set exit code to fail if either Python or Embedded tests failed if ($PYTHON_STATUS -ne 0 -or $CS_STATUS -ne 0 -or $CS_PERF_STATUS -ne 0) {