Skip to content

Improve path handling in run-tests #5877

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

Improve path handling in run-tests #5877

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ba0d7b5
tests/run-tests: Use absolute paths where possible.
stinos Apr 8, 2020
df7c9d4
tests/run-tests: Make test output directory configurable.
stinos Apr 7, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,7 @@ build-*/

# Test failure outputs
######################
tests/*.exp
tests/*.out
tests/results/*

# Python cache files
######################
Expand Down
58 changes: 34 additions & 24 deletions tests/run-tests
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,29 @@ import subprocess
import sys
import platform
import argparse
import inspect
import re
from glob import glob

# See stackoverflow.com/questions/2632199: __file__ nor sys.argv[0]
# are guaranteed to always work, this one should though.
BASEPATH = os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda: None)))

def base_path(*p):
return os.path.abspath(os.path.join(BASEPATH, *p)).replace('\\', '/')

# Tests require at least CPython 3.3. If your default python3 executable
# is of lower version, you can point MICROPY_CPYTHON3 environment var
# to the correct executable.
if os.name == 'nt':
CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3.exe')
MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../ports/windows/micropython.exe')
MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', base_path('../ports/windows/micropython.exe'))
else:
CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3')
MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../ports/unix/micropython')
MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', base_path('../ports/unix/micropython'))

# mpy-cross is only needed if --via-mpy command-line arg is passed
MPYCROSS = os.getenv('MICROPY_MPYCROSS', '../mpy-cross/mpy-cross')
MPYCROSS = os.getenv('MICROPY_MPYCROSS', base_path('../mpy-cross/mpy-cross'))

# For diff'ing test output
DIFF = os.getenv('MICROPY_DIFF', 'diff -u')
Expand Down Expand Up @@ -61,7 +69,7 @@ def run_micropython(pyb, args, test_file, is_special=False):
had_crash = False
if pyb is None:
# run on PC
if test_file.startswith(('cmdline/', 'feature_check/')) or test_file in special_tests:
if test_file.startswith(('cmdline/', base_path('feature_check/'))) or test_file in special_tests:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why does feature_check need base_path but cmdline doesn't?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because test_file will still be a relative path in most cases, except not for feature_check tests. In main() tests still get listed relative to the test directory, also see my comment on this PR above to why that is, and I added this to the hlpe text as well.

# special handling for tests of the unix cmdline program
is_special = True

Expand Down Expand Up @@ -215,10 +223,10 @@ def run_feature_check(pyb, args, base_path, test_file):
if pyb is not None and test_file.startswith("repl_"):
# REPL feature tests will not run via pyboard because they require prompt interactivity
return b""
return run_micropython(pyb, args, base_path + "/feature_check/" + test_file, is_special=True)
return run_micropython(pyb, args, base_path("feature_check", test_file), is_special=True)


def run_tests(pyb, tests, args, base_path="."):
def run_tests(pyb, tests, args, result_dir):
test_count = 0
testcase_count = 0
passed_count = 0
Expand All @@ -244,6 +252,10 @@ def run_tests(pyb, tests, args, base_path="."):
# If we're asked to --list-tests, we can't assume that there's a
# connection to target, so we can't run feature checks usefully.
if not (args.list_tests or args.write_exp):
# Even if we run completely different tests in a different directory,
# we need to access feature_checks from the same directory as the
# run-tests script itself so use base_path.

# Check if micropython.native is supported, and skip such tests if it's not
output = run_feature_check(pyb, args, base_path, 'native_check.py')
if output == b'CRASH':
Expand Down Expand Up @@ -307,7 +319,7 @@ def run_tests(pyb, tests, args, base_path="."):
upy_float_precision = int(upy_float_precision)
has_complex = run_feature_check(pyb, args, base_path, 'complex.py') == b'complex\n'
has_coverage = run_feature_check(pyb, args, base_path, 'coverage.py') == b'coverage\n'
cpy_byteorder = subprocess.check_output([CPYTHON3, base_path + '/feature_check/byteorder.py'])
cpy_byteorder = subprocess.check_output([CPYTHON3, base_path('feature_check/byteorder.py')])
skip_endian = (upy_byteorder != cpy_byteorder)

# These tests don't test slice explicitly but rather use it to perform the test
Expand Down Expand Up @@ -509,8 +521,8 @@ def run_tests(pyb, tests, args, base_path="."):

testcase_count += len(output_expected.splitlines())

filename_expected = test_basename + ".exp"
filename_mupy = test_basename + ".out"
filename_expected = os.path.join(result_dir, test_basename + ".exp")
filename_mupy = os.path.join(result_dir, test_basename + ".out")

if output_expected == output_mupy:
print("pass ", test_file)
Expand Down Expand Up @@ -563,10 +575,14 @@ def main():
formatter_class=argparse.RawDescriptionHelpFormatter,
description='''Run and manage tests for MicroPython.

Tests are discovered by scanning test directories for .py files or using the
specified test files. If test files nor directories are specified, the script
expects to be ran in the tests directory (where this file is located) and the
builtin tests suitable for the target platform are ran.
When running tests, run-tests compares the MicroPython output of the test with the output
produced by running the test through CPython unless a <test>.exp file is found, in which
case it is used as comparison.
If a test fails, run-tests produces a pair of <test>.out and <test>.exp files in the current
If a test fails, run-tests produces a pair of <test>.out and <test>.exp files in the result
directory with the MicroPython output and the expectations, respectively.
''',
epilog='''\
Expand All @@ -583,6 +599,7 @@ the last matching regex is used:
cmd_parser.add_argument('-u', '--user', default='micro', help='the telnet login username')
cmd_parser.add_argument('-p', '--password', default='python', help='the telnet login password')
cmd_parser.add_argument('-d', '--test-dirs', nargs='*', help='input test directories (if no files given)')
cmd_parser.add_argument('-r', '--result-dir', default=base_path('results'), help='directory for test results')
cmd_parser.add_argument('-e', '--exclude', action=append_filter, metavar='REGEX', dest='filters', help='exclude test by regex on path/name.py')
cmd_parser.add_argument('-i', '--include', action=append_filter, metavar='REGEX', dest='filters', help='include test by regex on path/name.py')
cmd_parser.add_argument('--write-exp', action='store_true', help='use CPython to generate .exp files to run tests w/o CPython')
Expand All @@ -598,20 +615,16 @@ the last matching regex is used:
args = cmd_parser.parse_args()

if args.print_failures:
os.chdir(os.path.abspath(os.path.dirname(__file__)))

for exp in glob("*.exp"):
testbase = os.path.basename(exp)[:-4]
for exp in glob(os.path.join(args.result_dir, "*.exp")):
testbase = exp[:-4]
print()
print("FAILURE {0}".format(testbase))
os.system("{0} {1}.exp {1}.out".format(DIFF, testbase))

sys.exit(0)

if args.clean_failures:
os.chdir(os.path.abspath(os.path.dirname(__file__)))

for f in glob("*.exp") + glob("*.out"):
for f in glob(os.path.join(args.result_dir, "*.exp")) + glob(os.path.join(args.result_dir, "*.out")):
os.remove(f)

sys.exit(0)
Expand All @@ -622,7 +635,7 @@ the last matching regex is used:
pyb = None
elif args.target in EXTERNAL_TARGETS:
global pyboard
sys.path.append('../tools')
sys.path.append(base_path('../tools'))
import pyboard
pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password)
pyb.enter_raw_repl()
Expand Down Expand Up @@ -659,14 +672,11 @@ the last matching regex is used:

if not args.keep_path:
# clear search path to make sure tests use only builtin modules and those in extmod
os.environ['MICROPYPATH'] = os.pathsep + '../extmod'
os.environ['MICROPYPATH'] = os.pathsep + base_path('../extmod')

# Even if we run completely different tests in a different directory,
# we need to access feature_check's from the same directory as the
# run-tests script itself.
base_path = os.path.dirname(sys.argv[0]) or "."
try:
res = run_tests(pyb, tests, args, base_path)
os.makedirs(args.result_dir, exist_ok=True)
res = run_tests(pyb, tests, args, args.result_dir)
finally:
if pyb:
pyb.close()
Expand Down