Skip to content

Commit 70d1c17

Browse files
vstinnervstinner
committed
gh-109566, regrtest: Add --fast-ci and --slow-ci options
* Add --fast-ci and --slow-ci options to libregrtest: * --fast-ci uses a default timeout of 10 minutes and "-u all,-cpu" (skip slowest tests). * --slow-ci uses a default timeout of 20 minues and "-u all" (run all tests). * Add "make fastcitest" target: similar to "make buildbottest", but use --fast-ci option instead of --slow-ci, and don't use TESTTIMEOUT. * "make buildbottest" now uses "--slow-ci". Remove now redundant options. * GitHub Actions workflow now uses "make fastcitest". Remove now redundant options. Just use -j0: detect the number of CPUs.
1 parent 2897142 commit 70d1c17

File tree

9 files changed

+100
-21
lines changed

9 files changed

+100
-21
lines changed

‎.github/workflows/build.yml

+5-5
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ jobs:
182182
- name: Display build info
183183
run: .\python.bat -m test.pythoninfo
184184
- name: Tests
185-
run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
185+
run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci
186186

187187
build_win_amd64:
188188
name: 'Windows (x64)'
@@ -201,7 +201,7 @@ jobs:
201201
- name: Display build info
202202
run: .\python.bat -m test.pythoninfo
203203
- name: Tests
204-
run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
204+
run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci
205205

206206
build_win_arm64:
207207
name: 'Windows (arm64)'
@@ -252,7 +252,7 @@ jobs:
252252
- name: Display build info
253253
run: make pythoninfo
254254
- name: Tests
255-
run: make buildbottest TESTOPTS="-j4 -uall,-cpu"
255+
run: make fastcitest
256256

257257
build_ubuntu:
258258
name: 'Ubuntu'
@@ -319,7 +319,7 @@ jobs:
319319
run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
320320
- name: Tests
321321
working-directory: ${{ env.CPYTHON_BUILDDIR }}
322-
run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
322+
run: xvfb-run make fastcitest
323323

324324
build_ubuntu_ssltests:
325325
name: 'Ubuntu SSL tests with OpenSSL'
@@ -535,7 +535,7 @@ jobs:
535535
- name: Display build info
536536
run: make pythoninfo
537537
- name: Tests
538-
run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
538+
run: xvfb-run make fastcitest
539539

540540
all-required-green: # This job does nothing and is only used for the branch protection
541541
name: All required checks pass

‎Doc/using/configure.rst

+3
Original file line numberDiff line numberDiff line change
@@ -967,6 +967,9 @@ Main Makefile targets
967967
* ``make buildbottest``: Build Python and run the Python test suite, the same
968968
way than buildbots test Python. Set ``TESTTIMEOUT`` variable (in seconds)
969969
to change the test timeout (1200 by default: 20 minutes).
970+
* ``make fastcitest``: Build Python and run the Python test suite using
971+
``--fast-ci`` option.
972+
test timeout (1200 by default: 20 minutes).
970973
* ``make install``: Build and install Python.
971974
* ``make regen-all``: Regenerate (almost) all generated files;
972975
``make regen-stdlib-module-names`` and ``autoconf`` must be run separately

‎Lib/test/libregrtest/cmdline.py

+40
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
import sys
55
from test.support import os_helper
66

7+
from .utils import MS_WINDOWS
8+
79

810
USAGE = """\
911
python -m test [options] [test_name1 [test_name2 ...]]
@@ -145,6 +147,7 @@
145147

146148
class Namespace(argparse.Namespace):
147149
def __init__(self, **kwargs) -> None:
150+
self.ci = False
148151
self.testdir = None
149152
self.verbose = 0
150153
self.quiet = False
@@ -209,6 +212,12 @@ def _create_parser():
209212
# We add help explicitly to control what argument group it renders under.
210213
group.add_argument('-h', '--help', action='help',
211214
help='show this help message and exit')
215+
group.add_argument('--fast-ci', action='store_true',
216+
help='Fast Continuous Integration (CI) mode used by '
217+
'GitHub Actions')
218+
group.add_argument('--slow-ci', action='store_true',
219+
help='Slow Continuous Integration (CI) mode used by '
220+
'buildbot workers')
212221
group.add_argument('--timeout', metavar='TIMEOUT', type=float,
213222
help='dump the traceback and exit if a test takes '
214223
'more than TIMEOUT seconds; disabled if TIMEOUT '
@@ -386,6 +395,37 @@ def _parse_args(args, **kwargs):
386395
parser.error("unrecognized arguments: %s" % arg)
387396
sys.exit(1)
388397

398+
# Continuous Integration (CI): common options for fast/slow CI modes
399+
if ns.slow_ci or ns.fast_ci:
400+
# Similar to options:
401+
#
402+
# -j0 --rerun -r --fail-env-changed --fail-rerun --slowest
403+
# --verbose3 --nowindows
404+
if ns.use_mp is None:
405+
ns.use_mp = 0
406+
ns.rerun = True
407+
ns.randomize = True
408+
ns.fail_env_changed = True
409+
ns.fail_rerun = True
410+
ns.print_slow = True
411+
ns.verbose3 = True
412+
if MS_WINDOWS:
413+
ns.nowindows = True # Silence alerts under Windows
414+
415+
# When --slow-ci and --fast-ci are present, --slow-ci has the priority
416+
if ns.slow_ci:
417+
# Similar to: -u "all" --timeout=1200
418+
if not ns.use:
419+
ns.use = [['all']]
420+
if ns.timeout is None:
421+
ns.timeout = 1200 # 20 minutes
422+
elif ns.fast_ci:
423+
# Similar to: -u "all,-cpu" --timeout=600
424+
if not ns.use:
425+
ns.use = [['all', '-cpu']]
426+
if ns.timeout is None:
427+
ns.timeout = 600 # 10 minutes
428+
389429
if ns.single and ns.fromfile:
390430
parser.error("-s and -f don't go together!")
391431
if ns.use_mp is not None and ns.trace:

‎Lib/test/test_regrtest.py

+31-1
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,9 @@
2323
from test import support
2424
from test.support import os_helper, TestStats, without_optimizer
2525
from test.libregrtest import cmdline
26-
from test.libregrtest import utils
26+
from test.libregrtest import main
2727
from test.libregrtest import setup
28+
from test.libregrtest import utils
2829
from test.libregrtest.utils import normalize_test_name
2930

3031
if not support.has_subprocess_support:
@@ -366,6 +367,35 @@ def test_unknown_option(self):
366367
self.checkError(['--unknown-option'],
367368
'unrecognized arguments: --unknown-option')
368369

370+
def check_ci_mode(self, args, use_resources):
371+
ns = cmdline._parse_args(args)
372+
if utils.MS_WINDOWS:
373+
self.assertTrue(ns.nowindows)
374+
375+
# Check Regrtest attributes which are more reliable than Namespace
376+
# which has an unclear API
377+
regrtest = main.Regrtest(ns)
378+
self.assertNotEqual(regrtest.num_workers, 0)
379+
self.assertTrue(regrtest.want_rerun)
380+
self.assertTrue(regrtest.randomize)
381+
self.assertIsNone(regrtest.random_seed)
382+
self.assertTrue(regrtest.fail_env_changed)
383+
self.assertTrue(regrtest.fail_rerun)
384+
self.assertTrue(regrtest.print_slowest)
385+
self.assertTrue(regrtest.output_on_failure)
386+
self.assertEqual(sorted(regrtest.use_resources), sorted(use_resources))
387+
388+
def test_fast_ci(self):
389+
args = ['--fast-ci']
390+
use_resources = sorted(cmdline.ALL_RESOURCES)
391+
use_resources.remove('cpu')
392+
self.check_ci_mode(args, use_resources)
393+
394+
def test_slow_ci(self):
395+
args = ['--slow-ci']
396+
use_resources = sorted(cmdline.ALL_RESOURCES)
397+
self.check_ci_mode(args, use_resources)
398+
369399

370400
@dataclasses.dataclass(slots=True)
371401
class Rerun:

‎Makefile.pre.in

+9-1
Original file line numberDiff line numberDiff line change
@@ -1892,7 +1892,15 @@ buildbottest: all
18921892
-@if which pybuildbot.identify >/dev/null 2>&1; then \
18931893
pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \
18941894
fi
1895-
$(TESTRUNNER) -j 1 -u all -W --slowest --fail-env-changed --fail-rerun --timeout=$(TESTTIMEOUT) $(TESTOPTS)
1895+
$(TESTRUNNER) --slow-ci --timeout=$(TESTTIMEOUT) $(TESTOPTS)
1896+
1897+
# Like buildbottest, but use --fast-ci option, instead of --slow-ci.
1898+
.PHONY: fastcitest
1899+
fastcitest: all
1900+
-@if which pybuildbot.identify >/dev/null 2>&1; then \
1901+
pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \
1902+
fi
1903+
$(TESTRUNNER) --fast-ci $(TESTOPTS)
18961904

18971905
# Like testall, but run Python tests with HOSTRUNNER directly.
18981906
.PHONY: hostrunnertest

‎Misc/NEWS.d/next/Build/2023-09-19-13-34-17.gh-issue-109566.RQAYRN.rst

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Add ``make fastcitest`` target: similar to ``make buildbottest``, but use
2+
``--fast-ci`` option instead of ``--slow-ci``, and don't use ``TESTTIMEOUT``
3+
option. Patch by Victor Stinner.

‎Misc/NEWS.d/next/Tests/2023-09-19-13-33-20.gh-issue-109566.aX0g9o.rst

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
regrtest: Add ``--fast-ci`` and ``--slow-ci`` options. ``--fast-ci`` uses a
2+
default timeout of 10 minutes and ``-u all,-cpu`` (skip slowest tests).
3+
``--slow-ci`` uses a default timeout of 20 minues and ``-u all`` (run all
4+
tests). Patch by Victor Stinner.

‎Tools/buildbot/test.bat

+4-3
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@ setlocal
55
set PATH=%PATH%;%SystemRoot%\SysNative\OpenSSH;%SystemRoot%\System32\OpenSSH
66
set here=%~dp0
77
set rt_opts=-q -d
8-
set regrtest_args=-j1
8+
set timeout=1200
9+
set regrtest_args=
910
set arm32_ssh=
1011

1112
:CheckOpts
@@ -23,7 +24,7 @@ if "%PROCESSOR_ARCHITECTURE%"=="ARM" if "%arm32_ssh%"=="true" goto NativeExecuti
2324
if "%arm32_ssh%"=="true" goto :Arm32Ssh
2425

2526
:NativeExecution
26-
call "%here%..\..\PCbuild\rt.bat" %rt_opts% -uall -rwW --slowest --timeout=1200 %regrtest_args%
27+
call "%here%..\..\PCbuild\rt.bat" %rt_opts% --slow-ci --timeout=%timeout% %regrtest_args%
2728
exit /b %ERRORLEVEL%
2829

2930
:Arm32Ssh
@@ -35,7 +36,7 @@ if NOT "%REMOTE_PYTHON_DIR:~-1,1%"=="\" (set REMOTE_PYTHON_DIR=%REMOTE_PYTHON_DI
3536

3637
set TEMP_ARGS=--temp %REMOTE_PYTHON_DIR%temp
3738

38-
set rt_args=%rt_opts% %dashU% -rwW --slowest --timeout=1200 %regrtest_args% %TEMP_ARGS%
39+
set rt_args=%rt_opts% --slow-ci %dashU% --timeout=%timeout% %regrtest_args% %TEMP_ARGS%
3940
ssh %SSH_SERVER% "set TEMP=%REMOTE_PYTHON_DIR%temp& cd %REMOTE_PYTHON_DIR% & %REMOTE_PYTHON_DIR%PCbuild\rt.bat" %rt_args%
4041
set ERR=%ERRORLEVEL%
4142
scp %SSH_SERVER%:"%REMOTE_PYTHON_DIR%test-results.xml" "%PYTHON_SOURCE%\test-results.xml"

‎Tools/scripts/run_tests.py

+1-11
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,6 @@ def is_multiprocess_flag(arg):
1818
return arg.startswith('-j') or arg.startswith('--multiprocess')
1919

2020

21-
def is_resource_use_flag(arg):
22-
return arg.startswith('-u') or arg.startswith('--use')
23-
2421
def is_python_flag(arg):
2522
return arg.startswith('-p') or arg.startswith('--python')
2623

@@ -56,20 +53,13 @@ def main(regrtest_args):
5653
args.extend(test.support.args_from_interpreter_flags())
5754

5855
args.extend(['-m', 'test', # Run the test suite
59-
'-r', # Randomize test order
60-
'-w', # Re-run failed tests in verbose mode
56+
'--fast-ci', # Fast Continuous Integration mode
6157
])
62-
if sys.platform == 'win32':
63-
args.append('-n') # Silence alerts under Windows
6458
if not any(is_multiprocess_flag(arg) for arg in regrtest_args):
6559
if cross_compile and hostrunner:
6660
# For now use only two cores for cross-compiled builds;
6761
# hostrunner can be expensive.
6862
args.extend(['-j', '2'])
69-
else:
70-
args.extend(['-j', '0']) # Use all CPU cores
71-
if not any(is_resource_use_flag(arg) for arg in regrtest_args):
72-
args.extend(['-u', 'all,-largefile,-audio,-gui'])
7363

7464
if cross_compile and hostrunner:
7565
# If HOSTRUNNER is set and -p/--python option is not given, then

0 commit comments

Comments
 (0)