Skip to content

gh-109162: Refactor libregrtest.RunTests #109177

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 9, 2023
Merged

gh-109162: Refactor libregrtest.RunTests #109177

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 32 additions & 25 deletions Lib/test/libregrtest/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
from test.libregrtest.cmdline import _parse_args, Namespace
from test.libregrtest.runtest import (
findtests, split_test_packages, run_single_test, abs_module_name,
PROGRESS_MIN_TIME, State, RunTests, TestResult,
PROGRESS_MIN_TIME, State, RunTests, TestResult, HuntRefleak,
FilterTuple, FilterDict, TestList)
from test.libregrtest.setup import setup_tests, setup_test_dir
from test.libregrtest.pgo import setup_pgo_tests
Expand Down Expand Up @@ -92,6 +92,14 @@ def __init__(self, ns: Namespace):
self.pgo_extended: bool = ns.pgo_extended
self.output_on_failure: bool = ns.verbose3
self.timeout: float | None = ns.timeout
self.verbose: bool = ns.verbose
self.quiet: bool = ns.quiet
if ns.huntrleaks:
self.hunt_refleak: HuntRefleak = HuntRefleak(*ns.huntrleaks)
else:
self.hunt_refleak = None
self.test_dir: str | None = ns.testdir
self.junit_filename: str | None = ns.xmlpath

# tests
self.tests = []
Expand Down Expand Up @@ -200,8 +208,7 @@ def log(self, line=''):
print(line, flush=True)

def display_progress(self, test_index, text):
quiet = self.ns.quiet
if quiet:
if self.quiet:
return

# "[ 51/405/1] test_tcl passed"
Expand All @@ -214,7 +221,6 @@ def display_progress(self, test_index, text):
def find_tests(self):
ns = self.ns
single = ns.single
test_dir = ns.testdir

if single:
self.next_single_filename = os.path.join(self.tmp_dir, 'pynexttest')
Expand Down Expand Up @@ -250,7 +256,8 @@ def find_tests(self):
exclude_tests.add(arg)
ns.args = []

alltests = findtests(testdir=test_dir, exclude=exclude_tests)
alltests = findtests(testdir=self.test_dir,
exclude=exclude_tests)

if not self.fromfile:
self.selected = self.tests or ns.args
Expand Down Expand Up @@ -298,14 +305,12 @@ def _list_cases(self, suite):
print(test.id())

def list_cases(self):
ns = self.ns
test_dir = ns.testdir
support.verbose = False
support.set_match_tests(self.match_tests, self.ignore_tests)

skipped = []
for test_name in self.selected:
module_name = abs_module_name(test_name, test_dir)
module_name = abs_module_name(test_name, self.test_dir)
try:
suite = unittest.defaultTestLoader.loadTestsFromName(module_name)
self._list_cases(suite)
Expand All @@ -331,7 +336,6 @@ def get_rerun_match(self, rerun_list) -> FilterDict:
def _rerun_failed_tests(self, need_rerun, runtests: RunTests):
# Configure the runner to re-run tests
ns = self.ns
ns.verbose = True
if ns.use_mp is None:
ns.use_mp = 1

Expand All @@ -349,6 +353,7 @@ def _rerun_failed_tests(self, need_rerun, runtests: RunTests):
runtests = runtests.copy(
tests=tuple(tests),
rerun=True,
verbose=True,
forever=False,
fail_fast=False,
match_tests_dict=match_tests_dict,
Expand Down Expand Up @@ -379,7 +384,6 @@ def rerun_failed_tests(self, need_rerun, runtests: RunTests):

def display_result(self, runtests):
pgo = runtests.pgo
quiet = self.ns.quiet
print_slow = self.ns.print_slow

# If running the test suite for PGO then no one cares about results.
Expand All @@ -398,7 +402,7 @@ def display_result(self, runtests):
print(count(len(omitted), "test"), "omitted:")
printlist(omitted)

if self.good and not quiet:
if self.good and not self.quiet:
print()
if (not self.bad
and not self.skipped
Expand All @@ -425,12 +429,12 @@ def display_result(self, runtests):
count(len(self.environment_changed), "test")))
printlist(self.environment_changed)

if self.skipped and not quiet:
if self.skipped and not self.quiet:
print()
print(count(len(self.skipped), "test"), "skipped:")
printlist(self.skipped)

if self.resource_denied and not quiet:
if self.resource_denied and not self.quiet:
print()
print(count(len(self.resource_denied), "test"), "skipped (resource denied):")
printlist(self.resource_denied)
Expand Down Expand Up @@ -684,7 +688,7 @@ def display_summary(self):
print(f"Result: {result}")

def save_xml_result(self):
if not self.ns.xmlpath and not self.testsuite_xml:
if not self.junit_filename and not self.testsuite_xml:
return

import xml.etree.ElementTree as ET
Expand All @@ -703,7 +707,7 @@ def save_xml_result(self):
for k, v in totals.items():
root.set(k, str(v))

xmlpath = os.path.join(os_helper.SAVEDCWD, self.ns.xmlpath)
xmlpath = os.path.join(os_helper.SAVEDCWD, self.junit_filename)
with open(xmlpath, 'wb') as f:
for s in ET.tostringlist(root):
f.write(s)
Expand Down Expand Up @@ -785,7 +789,7 @@ def main(self, tests: TestList | None = None):
ns = self.ns
self.tests = tests

if ns.xmlpath:
if self.junit_filename:
support.junit_xml_list = self.testsuite_xml = []

strip_py_suffix(ns.args)
Expand Down Expand Up @@ -844,16 +848,14 @@ def get_exitcode(self):
return exitcode

def action_run_tests(self):
if self.ns.huntrleaks:
warmup, repetitions, _ = self.ns.huntrleaks
if warmup < 3:
msg = ("WARNING: Running tests with --huntrleaks/-R and less than "
"3 warmup repetitions can give false positives!")
print(msg, file=sys.stdout, flush=True)
if self.hunt_refleak and self.hunt_refleak.warmups < 3:
msg = ("WARNING: Running tests with --huntrleaks/-R and "
"less than 3 warmup repetitions can give false positives!")
print(msg, file=sys.stdout, flush=True)

# For a partial run, we do not need to clutter the output.
if (self.want_header
or not(self.pgo or self.ns.quiet or self.ns.single
or not(self.pgo or self.quiet or self.ns.single
or self.tests or self.ns.args)):
self.display_header()

Expand All @@ -869,7 +871,12 @@ def action_run_tests(self):
pgo=self.pgo,
pgo_extended=self.pgo_extended,
output_on_failure=self.output_on_failure,
timeout=self.timeout)
timeout=self.timeout,
verbose=self.verbose,
quiet=self.quiet,
hunt_refleak=self.hunt_refleak,
test_dir=self.test_dir,
junit_filename=self.junit_filename)

setup_tests(runtests, self.ns)

Expand All @@ -892,7 +899,7 @@ def _main(self):
if self.want_wait:
input("Press any key to continue...")

setup_test_dir(self.ns.testdir)
setup_test_dir(self.test_dir)
self.find_tests()

exitcode = 0
Expand Down
23 changes: 14 additions & 9 deletions Lib/test/libregrtest/refleak.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from inspect import isabstract
from test import support
from test.support import os_helper
from test.libregrtest.runtest import HuntRefleak
from test.libregrtest.utils import clear_caches

try:
Expand All @@ -19,7 +20,9 @@ def _get_dump(cls):
cls._abc_negative_cache, cls._abc_negative_cache_version)


def dash_R(ns, test_name, test_func):
def runtest_refleak(test_name, test_func,
hunt_refleak: HuntRefleak,
quiet: bool):
"""Run a test multiple times, looking for reference leaks.

Returns:
Expand Down Expand Up @@ -62,9 +65,11 @@ def dash_R(ns, test_name, test_func):
def get_pooled_int(value):
return int_pool.setdefault(value, value)

nwarmup, ntracked, fname = ns.huntrleaks
fname = os.path.join(os_helper.SAVEDCWD, fname)
repcount = nwarmup + ntracked
warmups = hunt_refleak.warmups
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like this has broken all of the refleak buildbots: https://buildbot.python.org/all/#/builders/474/builds/1335/steps/5/logs/stdio

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

They are all failing in the same way on #109197

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah. A test is missing apparently. I wrote PR #109202 to fix the regression and add a test.

runs = hunt_refleak.runs
filename = hunt_refleak.filename
filename = os.path.join(os_helper.SAVEDCWD, filename)
repcount = warmups + runs

# Pre-allocate to ensure that the loop doesn't allocate anything new
rep_range = list(range(repcount))
Expand All @@ -78,7 +83,7 @@ def get_pooled_int(value):
# initialize variables to make pyflakes quiet
rc_before = alloc_before = fd_before = interned_before = 0

if not ns.quiet:
if not quiet:
print("beginning", repcount, "repetitions", file=sys.stderr)
print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
flush=True)
Expand All @@ -102,7 +107,7 @@ def get_pooled_int(value):
rc_after = gettotalrefcount() - interned_after * 2
fd_after = fd_count()

if not ns.quiet:
if not quiet:
print('.', end='', file=sys.stderr, flush=True)

rc_deltas[i] = get_pooled_int(rc_after - rc_before)
Expand All @@ -114,7 +119,7 @@ def get_pooled_int(value):
fd_before = fd_after
interned_before = interned_after

if not ns.quiet:
if not quiet:
print(file=sys.stderr)

# These checkers return False on success, True on failure
Expand Down Expand Up @@ -143,12 +148,12 @@ def check_fd_deltas(deltas):
(fd_deltas, 'file descriptors', check_fd_deltas)
]:
# ignore warmup runs
deltas = deltas[nwarmup:]
deltas = deltas[warmups:]
if checker(deltas):
msg = '%s leaked %s %s, sum=%s' % (
test_name, deltas, item_name, sum(deltas))
print(msg, file=sys.stderr, flush=True)
with open(fname, "a", encoding="utf-8") as refrep:
with open(filename, "a", encoding="utf-8") as refrep:
print(msg, file=refrep)
refrep.flush()
failed = True
Expand Down
Loading