From 7172c5dbb4e49d9a032e8657d033d552757b1c76 Mon Sep 17 00:00:00 2001 From: Wilko Nienhaus Date: Sun, 5 Sep 2021 15:44:47 +0000 Subject: [PATCH] all: Move frozen modules to virtual directory. This change "moves" frozen modules into a virtual directory called "|frozen". This allows controlling the search order for frozen modules, relative to other directories. Before this change, frozen modules were found via the current directory (i.e. the "" entry in sys.path). However, when a file is executed from the command line the first entry in sys.path (which is normally the empty string ""), is replaced by the base path of the executing file (correct behavior as per CPython). This made loading frozen modules fail. With this change, frozen modules have their own entry in sys.path, which also allows the user to control the order of search when looking for frozen modules. By default, the virtual "|frozen" directory is the last entry in sys.path (i.e. it gets searched last). Fixes issue #2322. --- ports/esp32/main.c | 3 +++ ports/esp8266/main.c | 3 +++ ports/mimxrt/main.c | 3 +++ ports/nrf/main.c | 5 +++++ ports/rp2/main.c | 3 +++ ports/stm32/main.c | 5 +++++ ports/teensy/main.c | 3 +++ ports/unix/main.c | 2 +- ports/windows/mpconfigport.h | 2 +- ports/zephyr/main.c | 3 +++ py/builtinimport.c | 15 +++++++++++---- py/modio.c | 7 +++++-- py/qstrdefs.h | 3 +++ shared/upytesthelper/upytesthelper.c | 3 +++ tests/run-tests.py | 2 +- tests/unix/extra_coverage.py.exp | 8 ++++---- 16 files changed, 57 insertions(+), 13 deletions(-) diff --git a/ports/esp32/main.c b/ports/esp32/main.c index ca0ab1488a125..3444be8922c16 100644 --- a/ports/esp32/main.c +++ b/ports/esp32/main.c @@ -143,6 +143,9 @@ void mp_task(void *pvParameter) { mp_obj_list_init(mp_sys_path, 0); mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_lib)); + #if MICROPY_MODULE_FROZEN + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__pipe_frozen)); + #endif mp_obj_list_init(mp_sys_argv, 0); readline_init0(); diff --git a/ports/esp8266/main.c b/ports/esp8266/main.c index 404188346cc00..c51e4f77fef0d 100644 --- a/ports/esp8266/main.c +++ b/ports/esp8266/main.c @@ -56,6 +56,9 @@ STATIC void mp_reset(void) { mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); // current dir (or base dir of the script) mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_lib)); mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_)); + #if MICROPY_MODULE_FROZEN + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__pipe_frozen)); + #endif mp_obj_list_init(mp_sys_argv, 0); #if MICROPY_EMIT_XTENSA || MICROPY_EMIT_INLINE_XTENSA extern void esp_native_code_init(void); diff --git a/ports/mimxrt/main.c b/ports/mimxrt/main.c index fa6237ca0750c..4b002be9a87ee 100644 --- a/ports/mimxrt/main.c +++ b/ports/mimxrt/main.c @@ -77,6 +77,9 @@ int main(void) { mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_path), 0); mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); + #if MICROPY_MODULE_FROZEN + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__pipe_frozen)); + #endif mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_argv), 0); #if MICROPY_PY_NETWORK mod_network_init(); diff --git a/ports/nrf/main.c b/ports/nrf/main.c index 254d9491c086f..d041428d101a3 100644 --- a/ports/nrf/main.c +++ b/ports/nrf/main.c @@ -252,6 +252,11 @@ int main(int argc, char **argv) { } #endif + #if MICROPY_MODULE_FROZEN + // add frozen virtual directory to path + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__pipe_frozen)); + #endif + // Main script is finished, so now go into REPL mode. // The REPL mode can change, or it can request a soft reset. int ret_code = 0; diff --git a/ports/rp2/main.c b/ports/rp2/main.c index 793444c9a92cf..954c94fe41dc9 100644 --- a/ports/rp2/main.c +++ b/ports/rp2/main.c @@ -103,6 +103,9 @@ int main(int argc, char **argv) { mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_path), 0); mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_lib)); + #if MICROPY_MODULE_FROZEN + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__pipe_frozen)); + #endif mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_argv), 0); // Initialise sub-systems. diff --git a/ports/stm32/main.c b/ports/stm32/main.c index 5d1ff2dcf5aad..9ed91e1fde372 100644 --- a/ports/stm32/main.c +++ b/ports/stm32/main.c @@ -561,6 +561,11 @@ void stm32_main(uint32_t reset_mode) { mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_flash_slash_lib)); } + #if MICROPY_MODULE_FROZEN + // add frozen virtual directory to path + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__pipe_frozen)); + #endif + // reset config variables; they should be set by boot.py MP_STATE_PORT(pyb_config_main) = MP_OBJ_NULL; diff --git a/ports/teensy/main.c b/ports/teensy/main.c index 6ebdcde21cbe7..c69de9624555e 100644 --- a/ports/teensy/main.c +++ b/ports/teensy/main.c @@ -271,6 +271,9 @@ int main(void) { mp_init(); mp_obj_list_init(mp_sys_path, 0); mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); // current dir (or base dir of the script) + #if MICROPY_MODULE_FROZEN + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__pipe_frozen)); + #endif mp_obj_list_init(mp_sys_argv, 0); readline_init0(); diff --git a/ports/unix/main.c b/ports/unix/main.c index 031bdd75d106e..b7669a171cd6f 100644 --- a/ports/unix/main.c +++ b/ports/unix/main.c @@ -498,7 +498,7 @@ MP_NOINLINE int main_(int argc, char **argv) { #ifdef MICROPY_PY_SYS_PATH_DEFAULT path = MICROPY_PY_SYS_PATH_DEFAULT; #else - path = "~/.micropython/lib:/usr/lib/micropython"; + path = "~/.micropython/lib:/usr/lib/micropython:|frozen"; #endif } size_t path_num = 1; // [0] is for current dir (or base dir of the script) diff --git a/ports/windows/mpconfigport.h b/ports/windows/mpconfigport.h index 8dd95e769d453..08225384e0eb3 100644 --- a/ports/windows/mpconfigport.h +++ b/ports/windows/mpconfigport.h @@ -88,7 +88,7 @@ #define MICROPY_PY_SYS_ATEXIT (1) #define MICROPY_PY_SYS_PLATFORM "win32" #ifndef MICROPY_PY_SYS_PATH_DEFAULT -#define MICROPY_PY_SYS_PATH_DEFAULT "~/.micropython/lib" +#define MICROPY_PY_SYS_PATH_DEFAULT "~/.micropython/lib;|frozen" #endif #define MICROPY_PY_SYS_MAXSIZE (1) #define MICROPY_PY_SYS_STDFILES (1) diff --git a/ports/zephyr/main.c b/ports/zephyr/main.c index 63190bd5edd67..ae7a3088d61cc 100644 --- a/ports/zephyr/main.c +++ b/ports/zephyr/main.c @@ -141,6 +141,9 @@ int real_main(void) { mp_init(); mp_obj_list_init(mp_sys_path, 0); mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); // current dir (or base dir of the script) + #if MICROPY_MODULE_FROZEN + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__pipe_frozen)); + #endif mp_obj_list_init(mp_sys_argv, 0); #ifdef CONFIG_USB diff --git a/py/builtinimport.c b/py/builtinimport.c index 08921f873b803..53b1f1084d635 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -47,6 +47,7 @@ #if MICROPY_ENABLE_EXTERNAL_IMPORT #define PATH_SEP_CHAR '/' +#define PATH_FROZEN "|frozen/" bool mp_obj_is_package(mp_obj_t module) { mp_obj_t dest[2]; @@ -58,9 +59,11 @@ bool mp_obj_is_package(mp_obj_t module) { // (whatever is available, if at all). STATIC mp_import_stat_t mp_import_stat_any(const char *path) { #if MICROPY_MODULE_FROZEN - mp_import_stat_t st = mp_frozen_stat(path); - if (st != MP_IMPORT_STAT_NO_EXIST) { - return st; + if (strncmp(path, PATH_FROZEN, strlen(PATH_FROZEN)) == 0) { + mp_import_stat_t st = mp_frozen_stat(path + strlen(PATH_FROZEN)); + if (st != MP_IMPORT_STAT_NO_EXIST) { + return st; + } } #endif return mp_import_stat(path); @@ -188,13 +191,17 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { // requested filename in the list of frozen module filenames. #if MICROPY_MODULE_FROZEN void *modref; - int frozen_type = mp_find_frozen_module(file_str, file->len, &modref); + int frozen_type = MP_FROZEN_NONE; + if (strncmp(file_str, PATH_FROZEN, strlen(PATH_FROZEN)) == 0) { + frozen_type = mp_find_frozen_module(file_str + strlen(PATH_FROZEN), file->len - strlen(PATH_FROZEN), &modref); + } #endif // If we support frozen str modules and the compiler is enabled, and we // found the filename in the list of frozen files, then load and execute it. #if MICROPY_MODULE_FROZEN_STR if (frozen_type == MP_FROZEN_STR) { + ((mp_lexer_t *)modref)->source_name = qstr_from_str(file_str); do_load_from_lexer(module_obj, modref); return; } diff --git a/py/modio.c b/py/modio.c index 7f0d13cdfaa0b..f13f21ca41a20 100644 --- a/py/modio.c +++ b/py/modio.c @@ -230,8 +230,11 @@ STATIC mp_obj_t resource_stream(mp_obj_t package_in, mp_obj_t path_in) { const char *path = mp_obj_str_get_data(path_in, &len); vstr_add_strn(&path_buf, path, len); - len = path_buf.len; - const char *data = mp_find_frozen_str(path_buf.buf, &len); + const char *data = NULL; + if (strncmp(path_buf.buf, "|frozen/", 8) == 0) { + len = path_buf.len - 8; + data = mp_find_frozen_str(path_buf.buf + 8, &len); // +8 skips |frozen/ + } if (data != NULL) { mp_obj_stringio_t *o = m_new_obj(mp_obj_stringio_t); o->base.type = &mp_type_bytesio; diff --git a/py/qstrdefs.h b/py/qstrdefs.h index 5b4e0dc48e053..2b47c32d809f8 100644 --- a/py/qstrdefs.h +++ b/py/qstrdefs.h @@ -47,6 +47,9 @@ Q({:#o}) Q({:#x}) #endif Q({:#b}) +#if MICROPY_MODULE_FROZEN +Q(|frozen) +#endif Q( ) Q(\n) Q(maximum recursion depth exceeded) diff --git a/shared/upytesthelper/upytesthelper.c b/shared/upytesthelper/upytesthelper.c index 326172be658c0..e129b01e9d15f 100644 --- a/shared/upytesthelper/upytesthelper.c +++ b/shared/upytesthelper/upytesthelper.c @@ -94,6 +94,9 @@ void upytest_execute_test(const char *src) { gc_init(heap_start, heap_end); mp_init(); mp_obj_list_init(mp_sys_path, 0); + #if MICROPY_MODULE_FROZEN + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__pipe_frozen)); + #endif mp_obj_list_init(mp_sys_argv, 0); nlr_buf_t nlr; diff --git a/tests/run-tests.py b/tests/run-tests.py index a8a31c0ae5c9f..8ec3a6b7c90a1 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -852,7 +852,7 @@ def main(): if not args.keep_path: # clear search path to make sure tests use only builtin modules and those in extmod - os.environ["MICROPYPATH"] = os.pathsep + base_path("../extmod") + os.environ["MICROPYPATH"] = os.pathsep + base_path("../extmod") + os.pathsep + "|frozen" try: os.makedirs(args.result_dir, exist_ok=True) diff --git a/tests/unix/extra_coverage.py.exp b/tests/unix/extra_coverage.py.exp index ea91813fc683f..37e190dd1263c 100644 --- a/tests/unix/extra_coverage.py.exp +++ b/tests/unix/extra_coverage.py.exp @@ -162,13 +162,13 @@ cpp None 5 (3, 'hellocpp') frzstr1 -frzstr1.py +|frozen/frzstr1.py frzmpy1 -frzmpy1.py +|frozen/frzmpy1.py frzstr_pkg1.__init__ -frzstr_pkg1/__init__.py 1 +|frozen/frzstr_pkg1/__init__.py 1 frzmpy_pkg1.__init__ -frzmpy_pkg1/__init__.py 1 +|frozen/frzmpy_pkg1/__init__.py 1 frzstr_pkg2.mod 1 frzmpy_pkg2.mod