micropython · dpgeorge · Aug 1, 2024 · Aug 1, 2024 · Aug 5, 2024 · Aug 5, 2024
diff --git a/ports/qemu-arm/Makefile b/ports/qemu-arm/Makefile
@@ -1,3 +1,5 @@
+TESTS_PROFILE = $(dir $(abspath $(firstword $(MAKEFILE_LIST))))/tests_profile.txt
+
 include ../../py/mkenv.mk
 -include mpconfigport.mk
 
@@ -6,6 +8,7 @@ QSTR_DEFS = qstrdefsport.h
 
 # MicroPython feature configurations
 MICROPY_ROM_TEXT_COMPRESSION ?= 1
+FROZEN_MANIFEST ?= "freeze('test-frzmpy')"
 
 # include py core make definitions
 include $(TOP)/py/py.mk
@@ -46,6 +49,8 @@ QEMU_EXTRA = -m 128M
 SRC_BOARD_O = shared/runtime/gchelper_generic.o
 # It's really armv7a but closest supported value is armv6.
 MPY_CROSS_FLAGS += -march=armv6
+# These don't work on Cortex-A9.
+TESTS_EXCLUDE = inlineasm/asmdiv.py inlineasm/asmspecialregs.py
 endif
 
 CROSS_COMPILE ?= arm-none-eabi-
@@ -87,6 +92,7 @@ SRC_RUN_C = \
 SRC_TEST_C = \
 	test_main.c \
 	lib/tinytest/tinytest.c \
+	shared/upytesthelper/upytesthelper.c \
 
 LIB_SRC_C += $(SRC_LIB_LIBM_C)
 LIB_SRC_C += $(SRC_LIB_LIBM_SQRT_SW_C)
@@ -123,4 +129,23 @@ $(BUILD)/firmware.elf: $(LDSCRIPT) $(ALL_OBJ_RUN)
 	$(Q)$(LD) $(LDFLAGS) -o $@ $(ALL_OBJ_RUN) $(LIBS)
 	$(Q)$(SIZE) $@
 
+$(BUILD)/firmware-test.elf: $(LDSCRIPT) $(ALL_OBJ_TEST)
+	$(Q)$(LD) $(LDFLAGS) -o $@ $(ALL_OBJ_TEST) $(LIBS)
+	$(Q)$(SIZE) $@
+
+$(BUILD)/test_main.o: $(BUILD)/genhdr/tests.h
+
+.PHONY: $(BUILD)/genhdr/tests.h
+$(BUILD)/genhdr/tests.h:
+	$(Q)echo "Generating $@"
+	$(Q)(cd $(TOP)/tests; ../tools/tinytest-codegen.py --target qemu-arm --profile $(TESTS_PROFILE) $(addprefix --exclude ,$(TESTS_EXCLUDE))) > $@
+
+$(BUILD)/lib/tinytest/tinytest.o: CFLAGS += -DNO_FORKING
+
+# Note: Using timeout(1) to handle cases where qemu hangs (e.g. this can happen with alignment errors).
+test: $(BUILD)/firmware-test.elf
+	timeout --foreground -k 5s 30s qemu-system-arm -machine $(BOARD) $(QEMU_EXTRA) -nographic -monitor null -semihosting -kernel $< > $(BUILD)/console.out
+	$(Q)tail -n2 $(BUILD)/console.out
+	$(Q)tail -n1 $(BUILD)/console.out | grep -q "status: 0"
+
 include $(TOP)/py/mkrules.mk
diff --git a/ports/qemu-arm/Makefile.test b/ports/qemu-arm/Makefile.test
diff --git a/ports/qemu-arm/README.md b/ports/qemu-arm/README.md
@@ -25,4 +25,4 @@ passed to the linker.
 
 To build and run image with builtin testsuite:
 
-    make -f Makefile.test test
+    make test
diff --git a/ports/qemu-arm/main.c b/ports/qemu-arm/main.c
@@ -32,6 +32,7 @@
 #include "py/stackctrl.h"
 #include "py/gc.h"
 #include "py/mperrno.h"
+#include "py/mphal.h"
 
 void do_str(const char *src, mp_parse_input_kind_t input_kind) {
     nlr_buf_t nlr;
@@ -70,3 +71,7 @@ void nlr_jump_fail(void *val) {
     printf("uncaught NLR\n");
     exit(1);
 }
+
+void qemu_print_strn(const char *str, size_t len) {
+    mp_hal_stdout_tx_strn_cooked(str, len);
+}
diff --git a/ports/qemu-arm/mpconfigport.h b/ports/qemu-arm/mpconfigport.h
@@ -78,8 +78,7 @@ typedef long mp_off_t;
 // We need an implementation of the log2 function which is not a macro.
 #define MP_NEED_LOG2 (1)
 
-#ifdef TEST
-#include "shared/upytesthelper/upytesthelper.h"
-#undef MP_PLAT_PRINT_STRN
-#define MP_PLAT_PRINT_STRN(str, len) upytest_output(str, len)
-#endif
+// All printing is passed through a custom function to check test output.
+#define MP_PLAT_PRINT_STRN(str, len) qemu_print_strn(str, len)
+
+void qemu_print_strn(const char *str, size_t len);
diff --git a/ports/qemu-arm/test_main.c b/ports/qemu-arm/test_main.c
@@ -33,6 +33,7 @@
 #include "py/gc.h"
 #include "py/mperrno.h"
 #include "shared/runtime/gchelper.h"
+#include "shared/upytesthelper/upytesthelper.h"
 #include "lib/tinytest/tinytest.h"
 #include "lib/tinytest/tinytest_macros.h"
 
@@ -64,3 +65,7 @@ void nlr_jump_fail(void *val) {
     printf("uncaught NLR\n");
     exit(1);
 }
+
+void qemu_print_strn(const char *str, size_t len) {
+    upytest_output(str, len);
+}
diff --git a/ports/qemu-riscv/Makefile b/ports/qemu-riscv/Makefile
@@ -1,3 +1,5 @@
+TESTS_PROFILE = $(dir $(abspath $(firstword $(MAKEFILE_LIST))))/tests_profile.txt
+
 include ../../py/mkenv.mk
 -include mpconfigport.mk
 
@@ -86,6 +88,7 @@ SRC_RUN_C = \
 SRC_TEST_C = \
 	test_main.c \
 	lib/tinytest/tinytest.c \
+	shared/upytesthelper/upytesthelper.c \
 
 LIB_SRC_C += $(SRC_LIB_LIBM_C)
 LIB_SRC_C += $(SRC_LIB_LIBM_SQRT_SW_C)
@@ -125,4 +128,27 @@ $(BUILD)/firmware.elf: $(LDSCRIPT) $(ALL_OBJ_RUN)
 	$(Q)$(LD) $(LDFLAGS) -o $@ $(ALL_OBJ_RUN) $(LIBS)
 	$(Q)$(SIZE) $@
 
+$(BUILD)/firmware-test.elf: $(LDSCRIPT) $(ALL_OBJ_TEST)
+	$(Q)$(LD) $(LDFLAGS) -o $@ $(ALL_OBJ_TEST) $(LIBS)
+	$(Q)$(SIZE) $@
+
+$(BUILD)/test_main.o: $(BUILD)/genhdr/tests.h
+
+.PHONY: $(BUILD)/genhdr/tests.h
+$(BUILD)/genhdr/tests.h:
+	$(Q)echo "Generating $@"
+	$(Q)(cd $(TOP)/tests; ../tools/tinytest-codegen.py --target=qemu-riscv --profile $(TESTS_PROFILE) $(addprefix --exclude ,$(TESTS_EXCLUDE))) > $@
+
+$(BUILD)/lib/tinytest/tinytest.o: CFLAGS += -DNO_FORKING
+
+# Note: Using timeout(1) to handle cases where qemu hangs (e.g. this can happen with alignment errors).
+test: $(BUILD)/firmware-test.elf
+	timeout --foreground -k 5s 60s qemu-system-riscv32 -machine $(BOARD) -bios none $(QEMU_EXTRA) -nographic -monitor null -semihosting -kernel $< > $(BUILD)/console.out
+	$(Q)tail -n2 $(BUILD)/console.out
+	$(Q)tail -n1 $(BUILD)/console.out | grep -q "status: 0"
+
+# `make debugtest` will block QEMU until a debugger is connected to port 1234.
+debugtest: $(BUILD)/firmware-test.elf
+	qemu-system-riscv32 -machine $(BOARD) -bios none $(QEMU_EXTRA) -nographic -monitor null -semihosting -serial mon:stdio -S -s -kernel $<
+
 include $(TOP)/py/mkrules.mk
diff --git a/ports/qemu-riscv/Makefile.test b/ports/qemu-riscv/Makefile.test
diff --git a/ports/qemu-riscv/README.md b/ports/qemu-riscv/README.md
@@ -26,4 +26,4 @@ Linux distribution's package manager, or independently packaged ones like
 
 To build and run the image with builtin testsuite:
 
-    make -f Makefile.test test
+    make test
diff --git a/ports/qemu-riscv/mpconfigport.h b/ports/qemu-riscv/mpconfigport.h
@@ -24,6 +24,7 @@
  * THE SOFTWARE.
  */
 
+#include <stddef.h>
 #include <stdint.h>
 
 // options to control how MicroPython is built
@@ -70,8 +71,7 @@ typedef long mp_off_t;
 // We need to provide a declaration/definition of alloca()
 #include <alloca.h>
 
-#ifdef TEST
-#include "shared/upytesthelper/upytesthelper.h"
-#undef MP_PLAT_PRINT_STRN
-#define MP_PLAT_PRINT_STRN(str, len) upytest_output(str, len)
-#endif
+// All printing is passed through a custom function to check test output.
+#define MP_PLAT_PRINT_STRN(str, len) qemu_print_strn(str, len)
+
+void qemu_print_strn(const char *str, size_t len);
diff --git a/ports/qemu-riscv/test_main.c b/ports/qemu-riscv/test_main.c
@@ -33,6 +33,7 @@
 #include "py/gc.h"
 #include "py/mperrno.h"
 #include "shared/runtime/gchelper.h"
+#include "shared/upytesthelper/upytesthelper.h"
 #include "lib/tinytest/tinytest.h"
 #include "lib/tinytest/tinytest_macros.h"
 
@@ -64,3 +65,7 @@ void nlr_jump_fail(void *val) {
     printf("uncaught NLR\n");
     exit(1);
 }
+
+void qemu_print_strn(const char *str, size_t len) {
+    upytest_output(str, len);
+}
diff --git a/ports/zephyr/make-bin-testsuite b/ports/zephyr/make-bin-testsuite
@@ -8,7 +8,6 @@
 # ./make-bin-testsuite BOARD=qemu_cortex_m3 run
 #
 
-(cd ../../tests; ./run-tests.py --write-exp)
 (cd ../../tests; ./run-tests.py --list-tests --target=minimal \
     -e async -e intbig -e int_big -e builtin_help -e memstats -e bytes_compare3 -e class_reverse_op \
     -e /set -e frozenset -e complex -e const -e native -e viper \

diff --git a/tests/run-tests.py b/tests/run-tests.py
@@ -441,7 +441,7 @@ def run_script_on_remote_target(self, args, test_file, is_special):
         return had_crash, output_mupy
 
 
-def run_tests(pyb, tests, args, result_dir, num_threads=1):
+def run_tests(pyb, tests, args, result_dir, num_threads=1, *, return_exp_dict=None):
     test_count = ThreadSafeCounter()
     testcase_count = ThreadSafeCounter()
     passed_count = ThreadSafeCounter()
@@ -467,7 +467,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
 
     # If we're asked to --list-tests, we can't assume that there's a
     # connection to target, so we can't run feature checks usefully.
-    if not (args.list_tests or args.write_exp):
+    if not (args.list_tests or args.write_exp or return_exp_dict is not None):
         # Even if we run completely different tests in a different directory,
         # we need to access feature_checks from the same directory as the
         # run-tests.py script itself so use base_path.
@@ -747,7 +747,7 @@ def run_tests(pyb, tests, args, result_dir, num_threads=1):
         skip_tests.add("micropython/schedule.py")  # native code doesn't check pending events
         skip_tests.add("stress/bytecode_limit.py")  # bytecode specific test
 
-    def run_one_test(test_file):
+    def run_one_test(test_file, *, output_dict=None):
         test_file = test_file.replace("\\", "/")
         test_file_abspath = os.path.abspath(test_file).replace("\\", "/")
 
@@ -796,8 +796,9 @@ def run_one_test(test_file):
             return
 
         if skip_it:
-            print("skip ", test_file)
-            skipped_tests.append(test_name)
+            if output_dict is None:
+                print("skip ", test_file)
+                skipped_tests.append(test_name)
             return
 
         # get expected output
@@ -823,6 +824,10 @@ def run_one_test(test_file):
         # canonical form for all host platforms is to use \n for end-of-line
         output_expected = output_expected.replace(b"\r\n", b"\n")
 
+        if output_dict is not None:
+            output_dict[test_file] = output_expected
+            return
+
         if args.write_exp:
             return
 
@@ -862,10 +867,13 @@ def run_one_test(test_file):
         pool.map(run_one_test, tests)
     else:
         for test in tests:
-            run_one_test(test)
+            run_one_test(test, output_dict=return_exp_dict)
+
+    if return_exp_dict is not None:
+        return True
 
     # Leave RESULTS_FILE untouched here for future runs.
-    if args.list_tests:
+    if args.list_tests or args.write_exp:
         return True
 
     print(
@@ -1138,22 +1146,6 @@ def main():
                     "cmdline",
                     "ports/unix",
                 )
-            elif args.target == "qemu-arm":
-                if not args.write_exp:
-                    raise ValueError("--target=qemu-arm must be used with --write-exp")
-                # Generate expected output files for qemu run.
-                # This list should match the test_dirs tuple in tinytest-codegen.py.
-                test_dirs += (
-                    "float",
-                    "inlineasm",
-                    "ports/qemu-arm",
-                )
-            elif args.target == "qemu-riscv":
-                if not args.write_exp:
-                    raise ValueError("--target=qemu-riscv must be used with --write-exp")
-                # Generate expected output files for qemu run.
-                # This list should match the test_dirs tuple in tinytest-codegen.py.
-                test_dirs += ("float",)
             elif args.target == "webassembly":
                 test_dirs += ("float", "ports/webassembly")
         else:

diff --git a/tools/ci.sh b/tools/ci.sh
@@ -258,10 +258,9 @@ function ci_qemu_arm_build {
     make ${MAKEOPTS} -C ports/qemu-arm submodules
     make ${MAKEOPTS} -C ports/qemu-arm CFLAGS_EXTRA=-DMP_ENDIANNESS_BIG=1
     make ${MAKEOPTS} -C ports/qemu-arm clean
-    make ${MAKEOPTS} -C ports/qemu-arm -f Makefile.test submodules
-    make ${MAKEOPTS} -C ports/qemu-arm -f Makefile.test test
-    make ${MAKEOPTS} -C ports/qemu-arm -f Makefile.test clean
-    make ${MAKEOPTS} -C ports/qemu-arm -f Makefile.test BOARD=sabrelite test
+    make ${MAKEOPTS} -C ports/qemu-arm test
+    make ${MAKEOPTS} -C ports/qemu-arm clean
+    make ${MAKEOPTS} -C ports/qemu-arm BOARD=sabrelite test
 }
 
 ########################################################################################
@@ -279,8 +278,7 @@ function ci_qemu_riscv_build {
     make ${MAKEOPTS} -C ports/qemu-riscv submodules
     make ${MAKEOPTS} -C ports/qemu-riscv
     make ${MAKEOPTS} -C ports/qemu-riscv clean
-    make ${MAKEOPTS} -C ports/qemu-riscv -f Makefile.test submodules
-    make ${MAKEOPTS} -C ports/qemu-riscv -f Makefile.test test
+    make ${MAKEOPTS} -C ports/qemu-riscv test
 }
 
 ########################################################################################
Original file line number	Diff line number	Diff line change
Expand Up		@@ -25,4 +25,4 @@ passed to the linker.

		To build and run image with builtin testsuite:

		make -f Makefile.test test
		make test
Original file line number	Diff line number	Diff line change
Expand Up		@@ -26,4 +26,4 @@ Linux distribution's package manager, or independently packaged ones like

		To build and run the image with builtin testsuite:

		make -f Makefile.test test
		make test