From ba0d7b5c48831607ca2615bd31b8961b23eba6d0 Mon Sep 17 00:00:00 2001 From: stijn Date: Wed, 8 Apr 2020 08:01:58 +0200 Subject: [PATCH 1/2] tests/run-tests: Use absolute paths where possible. Replace some usages of paths relative to the current working directory with absolute paths relative to the tests directory. Fixes and resulting changes: - default values of MICROPYTHON and MPYCROSS are absolute paths and always correct - likewise, the correct full paths for tools and extmod directories are appended to sys.path - printing/cleaning failures works properly since it expects the .exp and .out files in the tests directory which is also where they are written to now, plus no more need for changing directories This fixes #5872 and allows running custom tests which use run-tests without having to cd to the tests directory first, and the test output still is in the tests/ directory instead of the current working directory. Discovery of tests and all skip test logic based on paths relative to the current working directory remains unchanged which essentially means that for running most of MicroPython's own tests, run-tests must still be ran from within it's directory, so document that. --- tests/run-tests | 54 ++++++++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/tests/run-tests b/tests/run-tests index 102b0f7790e14..c2831614a3599 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -5,21 +5,29 @@ import subprocess import sys import platform import argparse +import inspect import re from glob import glob +# See stackoverflow.com/questions/2632199: __file__ nor sys.argv[0] +# are guaranteed to always work, this one should though. +BASEPATH = os.path.dirname(os.path.abspath(inspect.getsourcefile(lambda: None))) + +def base_path(*p): + return os.path.abspath(os.path.join(BASEPATH, *p)).replace('\\', '/') + # Tests require at least CPython 3.3. If your default python3 executable # is of lower version, you can point MICROPY_CPYTHON3 environment var # to the correct executable. if os.name == 'nt': CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3.exe') - MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../ports/windows/micropython.exe') + MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', base_path('../ports/windows/micropython.exe')) else: CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3') - MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../ports/unix/micropython') + MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', base_path('../ports/unix/micropython')) # mpy-cross is only needed if --via-mpy command-line arg is passed -MPYCROSS = os.getenv('MICROPY_MPYCROSS', '../mpy-cross/mpy-cross') +MPYCROSS = os.getenv('MICROPY_MPYCROSS', base_path('../mpy-cross/mpy-cross')) # For diff'ing test output DIFF = os.getenv('MICROPY_DIFF', 'diff -u') @@ -61,7 +69,7 @@ def run_micropython(pyb, args, test_file, is_special=False): had_crash = False if pyb is None: # run on PC - if test_file.startswith(('cmdline/', 'feature_check/')) or test_file in special_tests: + if test_file.startswith(('cmdline/', base_path('feature_check/'))) or test_file in special_tests: # special handling for tests of the unix cmdline program is_special = True @@ -215,10 +223,10 @@ def run_feature_check(pyb, args, base_path, test_file): if pyb is not None and test_file.startswith("repl_"): # REPL feature tests will not run via pyboard because they require prompt interactivity return b"" - return run_micropython(pyb, args, base_path + "/feature_check/" + test_file, is_special=True) + return run_micropython(pyb, args, base_path("feature_check", test_file), is_special=True) -def run_tests(pyb, tests, args, base_path="."): +def run_tests(pyb, tests, args): test_count = 0 testcase_count = 0 passed_count = 0 @@ -244,6 +252,10 @@ def run_tests(pyb, tests, args, base_path="."): # If we're asked to --list-tests, we can't assume that there's a # connection to target, so we can't run feature checks usefully. if not (args.list_tests or args.write_exp): + # Even if we run completely different tests in a different directory, + # we need to access feature_checks from the same directory as the + # run-tests script itself so use base_path. + # Check if micropython.native is supported, and skip such tests if it's not output = run_feature_check(pyb, args, base_path, 'native_check.py') if output == b'CRASH': @@ -307,7 +319,7 @@ def run_tests(pyb, tests, args, base_path="."): upy_float_precision = int(upy_float_precision) has_complex = run_feature_check(pyb, args, base_path, 'complex.py') == b'complex\n' has_coverage = run_feature_check(pyb, args, base_path, 'coverage.py') == b'coverage\n' - cpy_byteorder = subprocess.check_output([CPYTHON3, base_path + '/feature_check/byteorder.py']) + cpy_byteorder = subprocess.check_output([CPYTHON3, base_path('feature_check/byteorder.py')]) skip_endian = (upy_byteorder != cpy_byteorder) # These tests don't test slice explicitly but rather use it to perform the test @@ -509,8 +521,8 @@ def run_tests(pyb, tests, args, base_path="."): testcase_count += len(output_expected.splitlines()) - filename_expected = test_basename + ".exp" - filename_mupy = test_basename + ".out" + filename_expected = base_path(test_basename + ".exp") + filename_mupy = base_path(test_basename + ".out") if output_expected == output_mupy: print("pass ", test_file) @@ -563,6 +575,10 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter, description='''Run and manage tests for MicroPython. +Tests are discovered by scanning test directories for .py files or using the +specified test files. If test files nor directories are specified, the script +expects to be ran in the tests directory (where this file is located) and the +builtin tests suitable for the target platform are ran. When running tests, run-tests compares the MicroPython output of the test with the output produced by running the test through CPython unless a .exp file is found, in which case it is used as comparison. @@ -598,10 +614,8 @@ the last matching regex is used: args = cmd_parser.parse_args() if args.print_failures: - os.chdir(os.path.abspath(os.path.dirname(__file__))) - - for exp in glob("*.exp"): - testbase = os.path.basename(exp)[:-4] + for exp in glob(base_path("*.exp")): + testbase = exp[:-4] print() print("FAILURE {0}".format(testbase)) os.system("{0} {1}.exp {1}.out".format(DIFF, testbase)) @@ -609,9 +623,7 @@ the last matching regex is used: sys.exit(0) if args.clean_failures: - os.chdir(os.path.abspath(os.path.dirname(__file__))) - - for f in glob("*.exp") + glob("*.out"): + for f in glob(base_path("*.exp")) + glob(base_path("*.out")): os.remove(f) sys.exit(0) @@ -622,7 +634,7 @@ the last matching regex is used: pyb = None elif args.target in EXTERNAL_TARGETS: global pyboard - sys.path.append('../tools') + sys.path.append(base_path('../tools')) import pyboard pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password) pyb.enter_raw_repl() @@ -659,14 +671,10 @@ the last matching regex is used: if not args.keep_path: # clear search path to make sure tests use only builtin modules and those in extmod - os.environ['MICROPYPATH'] = os.pathsep + '../extmod' + os.environ['MICROPYPATH'] = os.pathsep + base_path('../extmod') - # Even if we run completely different tests in a different directory, - # we need to access feature_check's from the same directory as the - # run-tests script itself. - base_path = os.path.dirname(sys.argv[0]) or "." try: - res = run_tests(pyb, tests, args, base_path) + res = run_tests(pyb, tests, args) finally: if pyb: pyb.close() From df7c9d46389ba45e6c6698cb280b873e66a214af Mon Sep 17 00:00:00 2001 From: stijn Date: Tue, 7 Apr 2020 16:19:49 +0200 Subject: [PATCH 2/2] tests/run-tests: Make test output directory configurable. A configurable result directory is advantageous because it enables using a dedicated location, eventually outside of the source tree, instead of forcing the output files into a fixed directory which might also contain other files already. For that reason the default output directory also has been changed to tests/results/. --- .gitignore | 3 +-- tests/run-tests | 16 +++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 50bd30e877e87..c52f59eec74e6 100644 --- a/.gitignore +++ b/.gitignore @@ -27,8 +27,7 @@ build-*/ # Test failure outputs ###################### -tests/*.exp -tests/*.out +tests/results/* # Python cache files ###################### diff --git a/tests/run-tests b/tests/run-tests index c2831614a3599..49811d9b78845 100755 --- a/tests/run-tests +++ b/tests/run-tests @@ -226,7 +226,7 @@ def run_feature_check(pyb, args, base_path, test_file): return run_micropython(pyb, args, base_path("feature_check", test_file), is_special=True) -def run_tests(pyb, tests, args): +def run_tests(pyb, tests, args, result_dir): test_count = 0 testcase_count = 0 passed_count = 0 @@ -521,8 +521,8 @@ def run_tests(pyb, tests, args): testcase_count += len(output_expected.splitlines()) - filename_expected = base_path(test_basename + ".exp") - filename_mupy = base_path(test_basename + ".out") + filename_expected = os.path.join(result_dir, test_basename + ".exp") + filename_mupy = os.path.join(result_dir, test_basename + ".out") if output_expected == output_mupy: print("pass ", test_file) @@ -582,7 +582,7 @@ builtin tests suitable for the target platform are ran. When running tests, run-tests compares the MicroPython output of the test with the output produced by running the test through CPython unless a .exp file is found, in which case it is used as comparison. -If a test fails, run-tests produces a pair of .out and .exp files in the current +If a test fails, run-tests produces a pair of .out and .exp files in the result directory with the MicroPython output and the expectations, respectively. ''', epilog='''\ @@ -599,6 +599,7 @@ the last matching regex is used: cmd_parser.add_argument('-u', '--user', default='micro', help='the telnet login username') cmd_parser.add_argument('-p', '--password', default='python', help='the telnet login password') cmd_parser.add_argument('-d', '--test-dirs', nargs='*', help='input test directories (if no files given)') + cmd_parser.add_argument('-r', '--result-dir', default=base_path('results'), help='directory for test results') cmd_parser.add_argument('-e', '--exclude', action=append_filter, metavar='REGEX', dest='filters', help='exclude test by regex on path/name.py') cmd_parser.add_argument('-i', '--include', action=append_filter, metavar='REGEX', dest='filters', help='include test by regex on path/name.py') cmd_parser.add_argument('--write-exp', action='store_true', help='use CPython to generate .exp files to run tests w/o CPython') @@ -614,7 +615,7 @@ the last matching regex is used: args = cmd_parser.parse_args() if args.print_failures: - for exp in glob(base_path("*.exp")): + for exp in glob(os.path.join(args.result_dir, "*.exp")): testbase = exp[:-4] print() print("FAILURE {0}".format(testbase)) @@ -623,7 +624,7 @@ the last matching regex is used: sys.exit(0) if args.clean_failures: - for f in glob(base_path("*.exp")) + glob(base_path("*.out")): + for f in glob(os.path.join(args.result_dir, "*.exp")) + glob(os.path.join(args.result_dir, "*.out")): os.remove(f) sys.exit(0) @@ -674,7 +675,8 @@ the last matching regex is used: os.environ['MICROPYPATH'] = os.pathsep + base_path('../extmod') try: - res = run_tests(pyb, tests, args) + os.makedirs(args.result_dir, exist_ok=True) + res = run_tests(pyb, tests, args, args.result_dir) finally: if pyb: pyb.close()