tests/run-tests: Make test output directory configurable.

stinos · stinos · commit dd396ead4832 · 2020-08-25T11:24:47.000+02:00
A configurable result directory is advantageous because it enables
using a dedicated location, eventually outside of the source tree,
instead of forcing the output files into a fixed directory which might
also contain other files already. For that reason the default output
directory also has been changed to tests/results/.
diff --git a/.gitignore b/.gitignore
@@ -27,8 +27,7 @@ build-*/
 
 # Test failure outputs
 ######################
-tests/*.exp
-tests/*.out
+tests/results/*
 
 # Python cache files
 ######################
diff --git a/tests/run-tests b/tests/run-tests
@@ -226,7 +226,7 @@ def run_feature_check(pyb, args, base_path, test_file):
     return run_micropython(pyb, args, base_path("feature_check", test_file), is_special=True)
 
 
-def run_tests(pyb, tests, args):
+def run_tests(pyb, tests, args, result_dir):
     test_count = 0
     testcase_count = 0
     passed_count = 0
@@ -521,8 +521,8 @@ def run_tests(pyb, tests, args):
 
         testcase_count += len(output_expected.splitlines())
 
-        filename_expected = base_path(test_basename + ".exp")
-        filename_mupy = base_path(test_basename + ".out")
+        filename_expected = os.path.join(result_dir, test_basename + ".exp")
+        filename_mupy = os.path.join(result_dir, test_basename + ".out")
 
         if output_expected == output_mupy:
             print("pass ", test_file)
@@ -584,7 +584,7 @@ this directory.
 When running tests, run-tests compares the MicroPython output of the test with the output
 produced by running the test through CPython unless a <test>.exp file is found, in which
 case it is used as comparison.
-If a test fails, run-tests produces a pair of <test>.out and <test>.exp files in the current
+If a test fails, run-tests produces a pair of <test>.out and <test>.exp files in the result
 directory with the MicroPython output and the expectations, respectively.
 ''',
         epilog='''\
@@ -601,6 +601,7 @@ the last matching regex is used:
     cmd_parser.add_argument('-u', '--user', default='micro', help='the telnet login username')
     cmd_parser.add_argument('-p', '--password', default='python', help='the telnet login password')
     cmd_parser.add_argument('-d', '--test-dirs', nargs='*', help='input test directories (if no files given)')
+    cmd_parser.add_argument('-r', '--result-dir', default=base_path('results'), help='directory for test results')
     cmd_parser.add_argument('-e', '--exclude', action=append_filter, metavar='REGEX', dest='filters', help='exclude test by regex on path/name.py')
     cmd_parser.add_argument('-i', '--include', action=append_filter, metavar='REGEX', dest='filters', help='include test by regex on path/name.py')
     cmd_parser.add_argument('--write-exp', action='store_true', help='use CPython to generate .exp files to run tests w/o CPython')
@@ -616,7 +617,7 @@ the last matching regex is used:
     args = cmd_parser.parse_args()
 
     if args.print_failures:
-        for exp in glob(base_path("*.exp")):
+        for exp in glob(os.path.join(args.result_dir, "*.exp")):
             testbase = exp[:-4]
             print()
             print("FAILURE {0}".format(testbase))
@@ -625,7 +626,7 @@ the last matching regex is used:
         sys.exit(0)
 
     if args.clean_failures:
-        for f in glob(base_path("*.exp")) + glob(base_path("*.out")):
+        for f in glob(os.path.join(args.result_dir, "*.exp")) + glob(os.path.join(args.result_dir, "*.out")):
             os.remove(f)
 
         sys.exit(0)
@@ -676,7 +677,8 @@ the last matching regex is used:
         os.environ['MICROPYPATH'] = os.pathsep + base_path('../extmod')
 
     try:
-        res = run_tests(pyb, tests, args)
+        os.makedirs(args.result_dir, exist_ok=True)
+        res = run_tests(pyb, tests, args, args.result_dir)
     finally:
         if pyb:
             pyb.close()
