From 798f6ec71f6b86fcb82e72b7efb4d06784ad0172 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Fri, 14 Oct 2022 14:44:23 -0500 Subject: [PATCH 1/7] Switch tests to run on Adafruit_CircuitPython_asyncio (fails) This also depends on https://github.com/adafruit/Adafruit_CircuitPython_Ticks/pull/8 otherwise adafruit_ticks is unimportable and the tests are just skipped. Several of the tests fail, and one runs forever instead of terminating. We should fix our asyncio until the tests patch, then incorporate this change. --- tests/run-tests.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/run-tests.py b/tests/run-tests.py index 751b70886aa40..15e9899e897e8 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -870,8 +870,14 @@ def main(): if not args.keep_path: # clear search path to make sure tests use only builtin modules and those in extmod - os.environ["MICROPYPATH"] = os.pathsep + ".frozen" + os.pathsep + base_path("../extmod") - + os.environ["MICROPYPATH"] = os.pathsep.join( + [ + "", + ".frozen", + base_path("../frozen/Adafruit_CircuitPython_asyncio"), + base_path("../frozen/Adafruit_CircuitPython_Ticks"), + ] + ) try: os.makedirs(args.result_dir, exist_ok=True) res = run_tests(pyb, tests, args, args.result_dir, args.jobs) From 963a51487a3a37ae16e264adff4805fb0813c0af Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Fri, 14 Oct 2022 14:44:34 -0500 Subject: [PATCH 2/7] unix Makefile: easy wait to print test failures as diffs --- ports/unix/Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ports/unix/Makefile b/ports/unix/Makefile index c0d2bdfc63af7..4790e0eacf816 100644 --- a/ports/unix/Makefile +++ b/ports/unix/Makefile @@ -309,6 +309,10 @@ test_full: $(PROG) $(TOP)/tests/run-tests.py cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests.py --via-mpy $(RUN_TESTS_MPY_CROSS_FLAGS) --emit native -d basics float micropython cat $(TOP)/tests/basics/0prelim.py | ./$(PROG) | grep -q 'abc' +.PHONY: print-failures clean-failures +print-failures clean-failures: + ../../tests/run-tests.py --$@ + test_gcov: test_full gcov -o $(BUILD)/py $(TOP)/py/*.c gcov -o $(BUILD)/extmod $(TOP)/extmod/*.c From 4f190c9228dfeb7c42d6ff8e6a300653aae82cc9 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Fri, 14 Oct 2022 16:40:10 -0500 Subject: [PATCH 3/7] Handle tests that just won't stop --- tests/run-tests.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/run-tests.py b/tests/run-tests.py index 15e9899e897e8..ed0cf18e459d8 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -183,7 +183,17 @@ def send_get(what): # run the actual test try: - output_mupy = subprocess.check_output(cmdlist, stderr=subprocess.STDOUT) + result = subprocess.run( + cmdlist, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + check=True, + timeout=10, + ) + output_mupy = result.stdout + except subprocess.TimeoutExpired as er: + had_crash = True + output_mupy = (er.output or b"") + b"TIMEOUT" except subprocess.CalledProcessError as er: had_crash = True output_mupy = er.output + b"CRASH" @@ -869,7 +879,7 @@ def main(): tests = args.files if not args.keep_path: - # clear search path to make sure tests use only builtin modules and those in extmod + # clear search path to make sure tests use only builtin modules and those that can be frozen os.environ["MICROPYPATH"] = os.pathsep.join( [ "", From 6a8bc738ecf0e849cd1a973aeea3970262835231 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Fri, 14 Oct 2022 17:07:27 -0500 Subject: [PATCH 4/7] Make it easy to test just a subset of tests with e.g., 'make TEST_EXTRA="extmod/uasyncio*.py" test' --- ports/unix/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ports/unix/Makefile b/ports/unix/Makefile index 4790e0eacf816..5141beff4c15f 100644 --- a/ports/unix/Makefile +++ b/ports/unix/Makefile @@ -296,9 +296,10 @@ include $(TOP)/py/mkrules.mk .PHONY: test test_full +TEST_EXTRA ?= test: $(PROG) $(TOP)/tests/run-tests.py $(eval DIRNAME=ports/$(notdir $(CURDIR))) - cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests.py + cd $(TOP)/tests && MICROPY_MICROPYTHON=../$(DIRNAME)/$(PROG) ./run-tests.py $(TEST_EXTRA) test_full: $(PROG) $(TOP)/tests/run-tests.py $(eval DIRNAME=ports/$(notdir $(CURDIR))) From e590d27bf8990f2163ee50e271bc5ce18ee3d956 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 15 Oct 2022 10:48:05 -0500 Subject: [PATCH 5/7] Use CircuitPython _TICKS_PERIOD .. since Adafruit_CircuitPython_asyncio is hard-coded to this _TICKS_PERIOD not the one that would otherwise be used on Unix This fixes all the uasyncio test failures on Unix --- extmod/moduasyncio.c | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/extmod/moduasyncio.c b/extmod/moduasyncio.c index c4c4d36ad8f0e..19d9f2aef027b 100644 --- a/extmod/moduasyncio.c +++ b/extmod/moduasyncio.c @@ -70,24 +70,13 @@ STATIC mp_obj_t task_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf); /******************************************************************************/ // Ticks for task ordering in pairing heap -#if !CIRCUITPY || (defined(__unix__) || defined(__APPLE__)) -STATIC mp_obj_t ticks(void) { - return MP_OBJ_NEW_SMALL_INT(mp_hal_ticks_ms() & (MICROPY_PY_UTIME_TICKS_PERIOD - 1)); -} - -STATIC mp_int_t ticks_diff(mp_obj_t t1_in, mp_obj_t t0_in) { - mp_uint_t t0 = MP_OBJ_SMALL_INT_VALUE(t0_in); - mp_uint_t t1 = MP_OBJ_SMALL_INT_VALUE(t1_in); - mp_int_t diff = ((t1 - t0 + MICROPY_PY_UTIME_TICKS_PERIOD / 2) & (MICROPY_PY_UTIME_TICKS_PERIOD - 1)) - - MICROPY_PY_UTIME_TICKS_PERIOD / 2; - return diff; -} -#else #define _TICKS_PERIOD (1lu << 29) #define _TICKS_MAX (_TICKS_PERIOD - 1) #define _TICKS_HALFPERIOD (_TICKS_PERIOD >> 1) -#define ticks() supervisor_ticks_ms() +STATIC mp_obj_t ticks(void) { + return MP_OBJ_NEW_SMALL_INT(mp_hal_ticks_ms() & _TICKS_MAX); +} STATIC mp_int_t ticks_diff(mp_obj_t t1_in, mp_obj_t t0_in) { mp_uint_t t0 = MP_OBJ_SMALL_INT_VALUE(t0_in); @@ -95,7 +84,6 @@ STATIC mp_int_t ticks_diff(mp_obj_t t1_in, mp_obj_t t0_in) { mp_int_t diff = ((t1 - t0 + _TICKS_HALFPERIOD) & _TICKS_MAX) - _TICKS_HALFPERIOD; return diff; } -#endif STATIC int task_lt(mp_pairheap_t *n1, mp_pairheap_t *n2) { mp_obj_task_t *t1 = (mp_obj_task_t *)n1; From 55169e0b4db968c037fdd2ef6a359fa3a485a7dd Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Tue, 19 Jul 2022 23:18:22 +1000 Subject: [PATCH 6/7] extmod/uasyncio/task.py: Fix crash when non-awaited task is awaited. A task that has been sent to the loop's exception handler due to being re-scheduled twice will then subsequently cause a `raise None` if it is subsequently awaited. In the C version of task.py, this causes a segfault. This makes the await succeed (via raising StopIteration instead). Signed-off-by: Jim Mussared --- extmod/moduasyncio.c | 9 ++++- extmod/uasyncio/task.py | 8 +++- tests/extmod/uasyncio_task_exception.py | 42 +++++++++++++++++++++ tests/extmod/uasyncio_task_exception.py.exp | 6 +++ 4 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 tests/extmod/uasyncio_task_exception.py create mode 100644 tests/extmod/uasyncio_task_exception.py.exp diff --git a/extmod/moduasyncio.c b/extmod/moduasyncio.c index 19d9f2aef027b..c7d1753e2f3c9 100644 --- a/extmod/moduasyncio.c +++ b/extmod/moduasyncio.c @@ -290,8 +290,13 @@ STATIC mp_obj_t task_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { STATIC mp_obj_t task_iternext(mp_obj_t self_in) { mp_obj_task_t *self = MP_OBJ_TO_PTR(self_in); if (TASK_IS_DONE(self)) { - // Task finished, raise return value to caller so it can continue. - nlr_raise(self->data); + if (self->data == mp_const_none) { + // Task finished but has already been sent to the loop's exception handler. + mp_raise_StopIteration(MP_OBJ_NULL); + } else { + // Task finished, raise return value to caller so it can continue. + nlr_raise(self->data); + } } else { // Put calling task on waiting queue. mp_obj_t cur_task = mp_obj_dict_get(uasyncio_context, MP_OBJ_NEW_QSTR(MP_QSTR_cur_task)); diff --git a/extmod/uasyncio/task.py b/extmod/uasyncio/task.py index cd75a14a0934e..7a80b25208019 100644 --- a/extmod/uasyncio/task.py +++ b/extmod/uasyncio/task.py @@ -141,8 +141,12 @@ def __await__(self): def __next__(self): if not self.state: - # Task finished, raise return value to caller so it can continue. - raise self.data + if self.data is None: + # Task finished but has already been sent to the loop's exception handler. + raise StopIteration + else: + # Task finished, raise return value to caller so it can continue. + raise self.data else: # Put calling task on waiting queue. self.state.push_head(core.cur_task) diff --git a/tests/extmod/uasyncio_task_exception.py b/tests/extmod/uasyncio_task_exception.py new file mode 100644 index 0000000000000..6bf99dc93cb84 --- /dev/null +++ b/tests/extmod/uasyncio_task_exception.py @@ -0,0 +1,42 @@ +# In MicroPython, a non-awaited task with a pending exception will raise to +# the loop's exception handler the second time it is scheduled. This is +# because without reference counting we have no way to know when the task is +# truly "non awaited" -- i.e. we only know that it wasn't awaited in the time +# it took to be re-scheduled. + +# If the task _is_ subsequently awaited, then the await should succeed without +# raising. + +try: + import uasyncio as asyncio +except ImportError: + try: + import asyncio + except ImportError: + print("SKIP") + raise SystemExit + + +def custom_handler(loop, context): + print("exception handler", type(context["exception"]).__name__) + + +async def main(): + loop = asyncio.get_event_loop() + loop.set_exception_handler(custom_handler) + + async def task(): + print("raise") + raise OSError + + print("create") + t = asyncio.create_task(task()) + print("sleep 1") + await asyncio.sleep(0) + print("sleep 2") + await asyncio.sleep(0) + print("await") + await t # should not raise. + + +asyncio.run(main()) diff --git a/tests/extmod/uasyncio_task_exception.py.exp b/tests/extmod/uasyncio_task_exception.py.exp new file mode 100644 index 0000000000000..44dae61e1cbe0 --- /dev/null +++ b/tests/extmod/uasyncio_task_exception.py.exp @@ -0,0 +1,6 @@ +create +sleep 1 +raise +sleep 2 +exception handler OSError +await From f1b7bcbbecf84824d1991378853d21345b8b02f3 Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Wed, 19 Oct 2022 11:24:33 -0500 Subject: [PATCH 7/7] Run 'make update-frozen-modules' --- frozen/Adafruit_CircuitPython_Display_Shapes | 2 +- frozen/Adafruit_CircuitPython_Display_Text | 2 +- frozen/Adafruit_CircuitPython_RFM69 | 2 +- frozen/Adafruit_CircuitPython_SD | 2 +- frozen/Adafruit_CircuitPython_Ticks | 2 +- frozen/Adafruit_CircuitPython_asyncio | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/frozen/Adafruit_CircuitPython_Display_Shapes b/frozen/Adafruit_CircuitPython_Display_Shapes index 288a4f553d2f6..656be4d79196b 160000 --- a/frozen/Adafruit_CircuitPython_Display_Shapes +++ b/frozen/Adafruit_CircuitPython_Display_Shapes @@ -1 +1 @@ -Subproject commit 288a4f553d2f616331b5a042568c97b5bb0e44a7 +Subproject commit 656be4d79196b5f25ab9ebca731d448c5b3bdc17 diff --git a/frozen/Adafruit_CircuitPython_Display_Text b/frozen/Adafruit_CircuitPython_Display_Text index 41f06c33ef7a0..6cf9f3cf32e0c 160000 --- a/frozen/Adafruit_CircuitPython_Display_Text +++ b/frozen/Adafruit_CircuitPython_Display_Text @@ -1 +1 @@ -Subproject commit 41f06c33ef7a029210416ac61319698f5768e83e +Subproject commit 6cf9f3cf32e0c176c861de6356813ea4d08034d6 diff --git a/frozen/Adafruit_CircuitPython_RFM69 b/frozen/Adafruit_CircuitPython_RFM69 index a815f364badc0..96b4a05c8a225 160000 --- a/frozen/Adafruit_CircuitPython_RFM69 +++ b/frozen/Adafruit_CircuitPython_RFM69 @@ -1 +1 @@ -Subproject commit a815f364badc0dac3fe49e7d8206d00ce95894e1 +Subproject commit 96b4a05c8a225ad7ddc392be7fb69efebe151981 diff --git a/frozen/Adafruit_CircuitPython_SD b/frozen/Adafruit_CircuitPython_SD index c59df6b8c3d80..bb2fc8c157ee4 160000 --- a/frozen/Adafruit_CircuitPython_SD +++ b/frozen/Adafruit_CircuitPython_SD @@ -1 +1 @@ -Subproject commit c59df6b8c3d8006c290b63e95996e49e8e7124c0 +Subproject commit bb2fc8c157ee44869cde4cbc1ab20e6f31ac727f diff --git a/frozen/Adafruit_CircuitPython_Ticks b/frozen/Adafruit_CircuitPython_Ticks index 5436a0f27f33b..7832bbb5449d5 160000 --- a/frozen/Adafruit_CircuitPython_Ticks +++ b/frozen/Adafruit_CircuitPython_Ticks @@ -1 +1 @@ -Subproject commit 5436a0f27f33b364035ce499c020b274d77a25b7 +Subproject commit 7832bbb5449d55d8c7b731e4ff7490b801e94a9e diff --git a/frozen/Adafruit_CircuitPython_asyncio b/frozen/Adafruit_CircuitPython_asyncio index 3e8c50eb2230d..fbdb77d7127e7 160000 --- a/frozen/Adafruit_CircuitPython_asyncio +++ b/frozen/Adafruit_CircuitPython_asyncio @@ -1 +1 @@ -Subproject commit 3e8c50eb2230de7b8a20dedac33334b5dbdee21e +Subproject commit fbdb77d7127e7d6a8d3574440b0f790c94a28cf8