From 84e934059c90bfc7960b24a5fd068bc4189a8508 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Wed, 8 Dec 2021 14:18:39 +1100 Subject: [PATCH 1/8] py/modio.c: Remove io.resource_stream function. Not enabled on any port, and functionality is better suited to the micropython-lib implementation of pkg_resources. --- .../unix/variants/coverage/mpconfigvariant.h | 1 - py/modio.c | 47 ------------------- py/mpconfig.h | 11 ----- tests/unix/extra_coverage.py | 6 --- tests/unix/extra_coverage.py.exp | 1 - 5 files changed, 66 deletions(-) diff --git a/ports/unix/variants/coverage/mpconfigvariant.h b/ports/unix/variants/coverage/mpconfigvariant.h index 942117608fab1..f033dddb10550 100644 --- a/ports/unix/variants/coverage/mpconfigvariant.h +++ b/ports/unix/variants/coverage/mpconfigvariant.h @@ -51,7 +51,6 @@ #define MICROPY_PY_MATH_FACTORIAL (1) #define MICROPY_PY_URANDOM_EXTRA_FUNCS (1) #define MICROPY_PY_IO_BUFFEREDWRITER (1) -#define MICROPY_PY_IO_RESOURCE_STREAM (1) #define MICROPY_PY_UASYNCIO (1) #define MICROPY_PY_URE_DEBUG (1) #define MICROPY_PY_URE_MATCH_GROUPS (1) diff --git a/py/modio.c b/py/modio.c index 7f0d13cdfaa0b..e79d59e4e5555 100644 --- a/py/modio.c +++ b/py/modio.c @@ -204,50 +204,6 @@ STATIC const mp_obj_type_t mp_type_bufwriter = { }; #endif // MICROPY_PY_IO_BUFFEREDWRITER -#if MICROPY_PY_IO_RESOURCE_STREAM -STATIC mp_obj_t resource_stream(mp_obj_t package_in, mp_obj_t path_in) { - VSTR_FIXED(path_buf, MICROPY_ALLOC_PATH_MAX); - size_t len; - - // As an extension to pkg_resources.resource_stream(), we support - // package parameter being None, the path_in is interpreted as a - // raw path. - if (package_in != mp_const_none) { - // Pass "True" as sentinel value in fromlist to force returning of leaf module - mp_obj_t pkg = mp_import_name(mp_obj_str_get_qstr(package_in), mp_const_true, MP_OBJ_NEW_SMALL_INT(0)); - - mp_obj_t dest[2]; - mp_load_method_maybe(pkg, MP_QSTR___path__, dest); - if (dest[0] == MP_OBJ_NULL) { - mp_raise_TypeError(NULL); - } - - const char *path = mp_obj_str_get_data(dest[0], &len); - vstr_add_strn(&path_buf, path, len); - vstr_add_byte(&path_buf, '/'); - } - - const char *path = mp_obj_str_get_data(path_in, &len); - vstr_add_strn(&path_buf, path, len); - - len = path_buf.len; - const char *data = mp_find_frozen_str(path_buf.buf, &len); - if (data != NULL) { - mp_obj_stringio_t *o = m_new_obj(mp_obj_stringio_t); - o->base.type = &mp_type_bytesio; - o->vstr = m_new_obj(vstr_t); - vstr_init_fixed_buf(o->vstr, len + 1, (char *)data); - o->vstr->len = len; - o->pos = 0; - return MP_OBJ_FROM_PTR(o); - } - - mp_obj_t path_out = mp_obj_new_str(path_buf.buf, path_buf.len); - return mp_builtin_open(1, &path_out, (mp_map_t *)&mp_const_empty_map); -} -STATIC MP_DEFINE_CONST_FUN_OBJ_2(resource_stream_obj, resource_stream); -#endif - STATIC const mp_rom_map_elem_t mp_module_io_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_uio) }, // Note: mp_builtin_open_obj should be defined by port, it's not @@ -256,9 +212,6 @@ STATIC const mp_rom_map_elem_t mp_module_io_globals_table[] = { #if MICROPY_PY_IO_IOBASE { MP_ROM_QSTR(MP_QSTR_IOBase), MP_ROM_PTR(&mp_type_iobase) }, #endif - #if MICROPY_PY_IO_RESOURCE_STREAM - { MP_ROM_QSTR(MP_QSTR_resource_stream), MP_ROM_PTR(&resource_stream_obj) }, - #endif #if MICROPY_PY_IO_FILEIO { MP_ROM_QSTR(MP_QSTR_FileIO), MP_ROM_PTR(&mp_type_fileio) }, #if MICROPY_CPYTHON_COMPAT diff --git a/py/mpconfig.h b/py/mpconfig.h index 308a77666192e..86e3e0f34936d 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -1281,17 +1281,6 @@ typedef double mp_float_t; #define MICROPY_PY_IO_IOBASE (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) #endif -// Whether to provide "uio.resource_stream()" function with -// the semantics of CPython's pkg_resources.resource_stream() -// (allows to access binary resources in frozen source packages). -// Note that the same functionality can be achieved in "pure -// Python" by prepocessing binary resources into Python source -// and bytecode-freezing it (with a simple helper module available -// e.g. in micropython-lib). -#ifndef MICROPY_PY_IO_RESOURCE_STREAM -#define MICROPY_PY_IO_RESOURCE_STREAM (0) -#endif - // Whether to provide "io.FileIO" class #ifndef MICROPY_PY_IO_FILEIO #define MICROPY_PY_IO_FILEIO (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EXTRA_FEATURES) diff --git a/tests/unix/extra_coverage.py b/tests/unix/extra_coverage.py index b4808993a760e..8ea27cbf2f801 100644 --- a/tests/unix/extra_coverage.py +++ b/tests/unix/extra_coverage.py @@ -89,12 +89,6 @@ except ZeroDivisionError: print("ZeroDivisionError") -# test loading a resource from a frozen string -import uio - -buf = uio.resource_stream("frzstr_pkg2", "mod.py") -print(buf.read(21)) - # test for MP_QSTR_NULL regression from frzqstr import returns_NULL diff --git a/tests/unix/extra_coverage.py.exp b/tests/unix/extra_coverage.py.exp index ea91813fc683f..8ee233a712b4f 100644 --- a/tests/unix/extra_coverage.py.exp +++ b/tests/unix/extra_coverage.py.exp @@ -174,5 +174,4 @@ frzstr_pkg2.mod frzmpy_pkg2.mod 1 ZeroDivisionError -b'# test frozen package' NULL From 111a8289d7ff46631e62ec11b3b295e77250738c Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Wed, 8 Dec 2021 16:02:42 +1100 Subject: [PATCH 2/8] teensy: Use manifest.py instead of FROZEN_DIR. Signed-off-by: Jim Mussared --- ports/teensy/Makefile | 77 +++++++++++++++++++--------------------- ports/teensy/manifest.py | 1 + 2 files changed, 38 insertions(+), 40 deletions(-) create mode 100644 ports/teensy/manifest.py diff --git a/ports/teensy/Makefile b/ports/teensy/Makefile index d1ff421621568..fe46b0e061a23 100644 --- a/ports/teensy/Makefile +++ b/ports/teensy/Makefile @@ -6,6 +6,42 @@ QSTR_DEFS = qstrdefsport.h $(BUILD)/pins_qstr.h # MicroPython feature configurations MICROPY_ROM_TEXT_COMPRESSION ?= 1 +USE_FROZEN = 1 +USE_MEMZIP = 0 + +ifeq ($(USE_MEMZIP),1) +SRC_C += \ + shared/memzip/import.c \ + shared/memzip/lexermemzip.c \ + shared/memzip/memzip.c \ + +OBJ += $(BUILD)/memzip-files.o + +MAKE_MEMZIP = $(TOP)/shared/memzip/make-memzip.py +ifeq ($(MEMZIP_DIR),) +MEMZIP_DIR = memzip_files +endif + +$(BUILD)/memzip-files.o: $(BUILD)/memzip-files.c + $(call compile_c) + +$(BUILD)/memzip-files.c: $(shell find ${MEMZIP_DIR} -type f) + @$(ECHO) "Creating $@" + $(Q)$(PYTHON) $(MAKE_MEMZIP) --zip-file $(BUILD)/memzip-files.zip --c-file $@ $(MEMZIP_DIR) + +endif # USE_MEMZIP + +ifeq ($(USE_FROZEN),1) + +FROZEN_MANIFEST ?= "manifest.py" + +CFLAGS += -DMICROPY_MODULE_FROZEN_STR + +SRC_C += \ + lexerfrozen.c + +endif # USE_FROZEN + # include py core make definitions include $(TOP)/py/py.mk @@ -76,10 +112,7 @@ endif CFLAGS += -fdata-sections -ffunction-sections LDFLAGS += -Wl,--gc-sections -USE_FROZEN = 1 -USE_MEMZIP = 0 - -SRC_C = \ +SRC_C += \ hal_ftm.c \ hal_gpio.c \ help.c \ @@ -127,42 +160,6 @@ OBJ += $(BUILD)/pins_gen.o all: hex hex: $(BUILD)/micropython.hex -ifeq ($(USE_MEMZIP),1) -SRC_C += \ - shared/memzip/import.c \ - shared/memzip/lexermemzip.c \ - shared/memzip/memzip.c \ - -OBJ += $(BUILD)/memzip-files.o - -MAKE_MEMZIP = $(TOP)/shared/memzip/make-memzip.py -ifeq ($(MEMZIP_DIR),) -MEMZIP_DIR = memzip_files -endif - -$(BUILD)/memzip-files.o: $(BUILD)/memzip-files.c - $(call compile_c) - -$(BUILD)/memzip-files.c: $(shell find ${MEMZIP_DIR} -type f) - @$(ECHO) "Creating $@" - $(Q)$(PYTHON) $(MAKE_MEMZIP) --zip-file $(BUILD)/memzip-files.zip --c-file $@ $(MEMZIP_DIR) - -endif # USE_MEMZIP - -ifeq ($(USE_FROZEN),1) - -ifeq ($(FROZEN_DIR),) -FROZEN_DIR = memzip_files -endif - -CFLAGS += -DMICROPY_MODULE_FROZEN_STR - -SRC_C += \ - lexerfrozen.c \ - $(BUILD)/frozen.c - -endif # USE_FROZEN - ifeq ($(ARDUINO),) post_compile: $(BUILD)/micropython.hex $(ECHO) "Please define ARDUINO (where TeensyDuino is installed)" diff --git a/ports/teensy/manifest.py b/ports/teensy/manifest.py new file mode 100644 index 0000000000000..1fefd686878d8 --- /dev/null +++ b/ports/teensy/manifest.py @@ -0,0 +1 @@ +freeze_as_str("$(PORT_DIR)/memzip_files") From 6e22753e8492a3e320376250ae9d769993ec58c5 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Wed, 8 Dec 2021 16:04:11 +1100 Subject: [PATCH 3/8] top: Remove support for FROZEN_DIR, FROZEN_MPY_DIR. This has been deprecated for over two years in favour of manifest.py. Signed-off-by: Jim Mussared --- ports/esp8266/Makefile | 7 +---- ports/javascript/Makefile | 7 ----- ports/nrf/Makefile | 16 ++--------- ports/nrf/README.md | 30 -------------------- ports/stm32/Makefile | 11 ++------ ports/unix/Makefile | 9 ++---- py/mkrules.mk | 58 ++++----------------------------------- py/py.mk | 10 ------- 8 files changed, 13 insertions(+), 135 deletions(-) diff --git a/ports/esp8266/Makefile b/ports/esp8266/Makefile index e771d8f7d3460..d0afe73ea8b04 100644 --- a/ports/esp8266/Makefile +++ b/ports/esp8266/Makefile @@ -27,8 +27,6 @@ AXTLS_DEFS_EXTRA = -Dabort=abort_ -DRT_MAX_PLAIN_LENGTH=1024 -DRT_EXTRA=4096 BTREE_DEFS_EXTRA = -DDEFPSIZE=1024 -DMINCACHE=3 FROZEN_MANIFEST ?= boards/manifest.py -FROZEN_DIR ?= -FROZEN_MPY_DIR ?= # include py core make definitions include $(TOP)/py/py.mk @@ -196,12 +194,9 @@ $(BUILD)/uart.o: $(CONFVARS_FILE) FROZEN_EXTRA_DEPS = $(CONFVARS_FILE) -ifneq ($(FROZEN_MANIFEST)$(FROZEN_MPY_DIR),) +ifneq ($(FROZEN_MANIFEST),) CFLAGS += -DMICROPY_MODULE_FROZEN_MPY CFLAGS += -DMICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool -endif - -ifneq ($(FROZEN_MANIFEST)$(FROZEN_DIR),) CFLAGS += -DMICROPY_MODULE_FROZEN_STR endif diff --git a/ports/javascript/Makefile b/ports/javascript/Makefile index 3fac114f5ed2b..ba2680b2a85aa 100644 --- a/ports/javascript/Makefile +++ b/ports/javascript/Makefile @@ -17,13 +17,6 @@ CFLAGS += -std=c99 -Wall -Werror -Wdouble-promotion -Wfloat-conversion CFLAGS += -O3 -DNDEBUG CFLAGS += $(INC) -ifneq ($(FROZEN_MPY_DIR),) -# To use frozen bytecode, put your .py files in a subdirectory (eg frozen/) and -# then invoke make with FROZEN_MPY_DIR=frozen (be sure to build from scratch). -CFLAGS += -DMICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool -CFLAGS += -DMICROPY_MODULE_FROZEN_MPY -endif - SRC_SHARED = $(addprefix shared/,\ runtime/interrupt_char.c \ runtime/stdout_helpers.c \ diff --git a/ports/nrf/Makefile b/ports/nrf/Makefile index 423323a9c6a2c..c32e09d2c9ce8 100644 --- a/ports/nrf/Makefile +++ b/ports/nrf/Makefile @@ -340,11 +340,6 @@ DRIVERS_SRC_C += $(addprefix modules/,\ SRC_C += \ device/startup_$(MCU_SUB_VARIANT).c \ -ifneq ($(FROZEN_MPY_DIR),) -FROZEN_MPY_PY_FILES := $(shell find -L $(FROZEN_MPY_DIR) -type f -name '*.py') -FROZEN_MPY_MPY_FILES := $(addprefix $(BUILD)/,$(FROZEN_MPY_PY_FILES:.py=.mpy)) -endif - LIBGCC_FILE_NAME = $(shell $(CC) $(CFLAGS) -print-libgcc-file-name) LIBS += -L $(dir $(LIBGCC_FILE_NAME)) -lgcc @@ -542,17 +537,10 @@ GEN_PINS_QSTR = $(BUILD)/pins_qstr.h GEN_PINS_AF_CONST = $(HEADER_BUILD)/pins_af_const.h GEN_PINS_AF_PY = $(BUILD)/pins_af.py -ifneq ($(FROZEN_MANIFEST)$(FROZEN_DIR),) -# To use frozen source modules, put your .py files in a subdirectory (eg scripts/) -# and then invoke make with FROZEN_DIR=scripts (be sure to build from scratch). -CFLAGS += -DMICROPY_MODULE_FROZEN_STR -endif - -ifneq ($(FROZEN_MANIFEST)$(FROZEN_MPY_DIR),) -# To use frozen bytecode, put your .py files in a subdirectory (eg frozen/) and -# then invoke make with FROZEN_MPY_DIR=frozen (be sure to build from scratch). +ifneq ($(FROZEN_MANIFEST),) CFLAGS += -DMICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool CFLAGS += -DMICROPY_MODULE_FROZEN_MPY +CFLAGS += -DMICROPY_MODULE_FROZEN_STR endif $(PY_BUILD)/nlr%.o: CFLAGS += -Os -fno-lto diff --git a/ports/nrf/README.md b/ports/nrf/README.md index aa8968ff92666..6ffed7266e1f6 100644 --- a/ports/nrf/README.md +++ b/ports/nrf/README.md @@ -96,36 +96,6 @@ The **make sd** will trigger a flash of the bluetooth stack before that applicat Note: further tuning of features to include in bluetooth or even setting up the device to use REPL over Bluetooth can be configured in the `bluetooth_conf.h`. -## Compile with frozen modules - -Frozen modules are Python modules compiled to bytecode and added to the firmware -image, as part of MicroPython. They can be imported as usual, using the `import` -statement. The advantage is that frozen modules use a lot less RAM as the -bytecode is stored in flash, not in RAM like when importing from a filesystem. -Also, frozen modules are available even when no filesystem is present to import -from. - -To use frozen modules, put them in a directory (e.g. `freeze/`) and supply -`make` with the given directory. For example: - - make BOARD=pca10040 FROZEN_MPY_DIR=freeze - -## Compile with freeze manifest - -Freeze manifests can be used by definining `FROZEN_MANIFEST` pointing to a -`manifest.py`. This can either be done by a `make` invocation or by defining -it in the specific target board's `mpconfigboard.mk`. - -For example: - - make BOARD=pca10040 FROZEN_MANIFEST=path/to/manifest.py - -In case of using the target board's makefile, add a line similar to this: - - FROZEN_MANIFEST ?= $(BOARD_DIR)/manifest.py - -In these two examples, the manual `make` invocation will have precedence. - ## Enable MICROPY_VFS_FAT As the `oofatfs` module is not having header guards that can exclude the implementation compile time, this port provides a flag to enable it explicitly. The MICROPY_VFS_FAT is by default set to 0 and has to be set to 1 if `oofatfs` files should be compiled. This will be in addition of setting `MICROPY_VFS` in mpconfigport.h. diff --git a/ports/stm32/Makefile b/ports/stm32/Makefile index c3b281650afd8..59b387933bc3c 100644 --- a/ports/stm32/Makefile +++ b/ports/stm32/Makefile @@ -591,17 +591,10 @@ $(TOP)/lib/stm32lib/README.md: $(ECHO) "stm32lib submodule not found, fetching it now..." (cd $(TOP) && git submodule update --init lib/stm32lib) -ifneq ($(FROZEN_MANIFEST)$(FROZEN_DIR),) -# To use frozen source modules, put your .py files in a subdirectory (eg scripts/) -# and then invoke make with FROZEN_DIR=scripts (be sure to build from scratch). -CFLAGS += -DMICROPY_MODULE_FROZEN_STR -endif - -ifneq ($(FROZEN_MANIFEST)$(FROZEN_MPY_DIR),) -# To use frozen bytecode, put your .py files in a subdirectory (eg frozen/) and -# then invoke make with FROZEN_MPY_DIR=frozen (be sure to build from scratch). +ifneq ($(FROZEN_MANIFEST),) CFLAGS += -DMICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool CFLAGS += -DMICROPY_MODULE_FROZEN_MPY +CFLAGS += -DMICROPY_MODULE_FROZEN_STR endif define RUN_DFU diff --git a/ports/unix/Makefile b/ports/unix/Makefile index f829838ab31f7..cd8bb379c4488 100644 --- a/ports/unix/Makefile +++ b/ports/unix/Makefile @@ -13,10 +13,8 @@ include ../../py/mkenv.mk -include mpconfigport.mk include $(VARIANT_DIR)/mpconfigvariant.mk -# use FROZEN_MANIFEST for new projects, others are legacy +# Use the default frozen manifest, variants may override this. FROZEN_MANIFEST ?= variants/manifest.py -FROZEN_DIR = -FROZEN_MPY_DIR = # This should be configured by the mpconfigvariant.mk PROG ?= micropython @@ -263,15 +261,12 @@ SRC_QSTR += $(SRC_C) $(SRC_CXX) $(SHARED_SRC_C) $(EXTMOD_SRC_C) # SRC_QSTR SRC_QSTR_AUTO_DEPS += -ifneq ($(FROZEN_MANIFEST)$(FROZEN_MPY_DIR),) +ifneq ($(FROZEN_MANIFEST),) # To use frozen code create a manifest.py file with a description of files to # freeze, then invoke make with FROZEN_MANIFEST=manifest.py (be sure to build from scratch). CFLAGS += -DMICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool CFLAGS += -DMICROPY_MODULE_FROZEN_MPY CFLAGS += -DMPZ_DIG_SIZE=16 # force 16 bits to work on both 32 and 64 bit archs -endif - -ifneq ($(FROZEN_MANIFEST)$(FROZEN_DIR),) CFLAGS += -DMICROPY_MODULE_FROZEN_STR endif diff --git a/py/mkrules.mk b/py/mkrules.mk index d0c0a53c262e5..963d70c2c38b4 100644 --- a/py/mkrules.mk +++ b/py/mkrules.mk @@ -142,43 +142,18 @@ $(MICROPY_MPYCROSS_DEPENDENCY): $(MAKE) -C $(dir $@) endif -ifneq ($(FROZEN_MANIFEST),) -# to build frozen_content.c from a manifest -$(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h | $(MICROPY_MPYCROSS_DEPENDENCY) - $(Q)$(MAKE_MANIFEST) -o $@ -v "MPY_DIR=$(TOP)" -v "MPY_LIB_DIR=$(MPY_LIB_DIR)" -v "PORT_DIR=$(shell pwd)" -v "BOARD_DIR=$(BOARD_DIR)" -b "$(BUILD)" $(if $(MPY_CROSS_FLAGS),-f"$(MPY_CROSS_FLAGS)",) --mpy-tool-flags="$(MPY_TOOL_FLAGS)" $(FROZEN_MANIFEST) - ifneq ($(FROZEN_DIR),) -$(error FROZEN_DIR cannot be used in conjunction with FROZEN_MANIFEST) +$(error Support for FROZEN_DIR was removed. Please use manifest.py instead, see https://docs.micropython.org/en/latest/reference/manifest.html) endif ifneq ($(FROZEN_MPY_DIR),) -$(error FROZEN_MPY_DIR cannot be used in conjunction with FROZEN_MANIFEST) -endif -endif - -ifneq ($(FROZEN_DIR),) -$(info Warning: FROZEN_DIR is deprecated in favour of FROZEN_MANIFEST) -$(BUILD)/frozen.c: $(wildcard $(FROZEN_DIR)/*) $(HEADER_BUILD) $(FROZEN_EXTRA_DEPS) - $(ECHO) "GEN $@" - $(Q)$(MAKE_FROZEN) $(FROZEN_DIR) > $@ +$(error Support for FROZEN_MPY_DIR was removed. Please use manifest.py instead, see https://docs.micropython.org/en/latest/reference/manifest.html) endif -ifneq ($(FROZEN_MPY_DIR),) -$(info Warning: FROZEN_MPY_DIR is deprecated in favour of FROZEN_MANIFEST) -# make a list of all the .py files that need compiling and freezing -FROZEN_MPY_PY_FILES := $(shell find -L $(FROZEN_MPY_DIR) -type f -name '*.py' | $(SED) -e 's=^$(FROZEN_MPY_DIR)/==') -FROZEN_MPY_MPY_FILES := $(addprefix $(BUILD)/frozen_mpy/,$(FROZEN_MPY_PY_FILES:.py=.mpy)) - -# to build .mpy files from .py files -$(BUILD)/frozen_mpy/%.mpy: $(FROZEN_MPY_DIR)/%.py | $(MICROPY_MPYCROSS_DEPENDENCY) - @$(ECHO) "MPY $<" - $(Q)$(MKDIR) -p $(dir $@) - $(Q)$(MICROPY_MPYCROSS) -o $@ -s $(<:$(FROZEN_MPY_DIR)/%=%) $(MPY_CROSS_FLAGS) $< - -# to build frozen_mpy.c from all .mpy files -$(BUILD)/frozen_mpy.c: $(FROZEN_MPY_MPY_FILES) $(BUILD)/genhdr/qstrdefs.generated.h - @$(ECHO) "GEN $@" - $(Q)$(MPY_TOOL) -f -q $(BUILD)/genhdr/qstrdefs.preprocessed.h $(FROZEN_MPY_MPY_FILES) > $@ +ifneq ($(FROZEN_MANIFEST),) +# to build frozen_content.c from a manifest +$(BUILD)/frozen_content.c: FORCE $(BUILD)/genhdr/qstrdefs.generated.h | $(MICROPY_MPYCROSS_DEPENDENCY) + $(Q)$(MAKE_MANIFEST) -o $@ -v "MPY_DIR=$(TOP)" -v "MPY_LIB_DIR=$(MPY_LIB_DIR)" -v "PORT_DIR=$(shell pwd)" -v "BOARD_DIR=$(BOARD_DIR)" -b "$(BUILD)" $(if $(MPY_CROSS_FLAGS),-f"$(MPY_CROSS_FLAGS)",) --mpy-tool-flags="$(MPY_TOOL_FLAGS)" $(FROZEN_MANIFEST) endif ifneq ($(PROG),) @@ -234,27 +209,6 @@ clean: $(RM) -rf $(BUILD) $(CLEAN_EXTRA) .PHONY: clean -# Clean every non-git file from FROZEN_DIR/FROZEN_MPY_DIR, but making a backup. -# We run rmdir below to avoid empty backup dir (it will silently fail if backup -# is non-empty). -clean-frozen: - if [ -n "$(FROZEN_MPY_DIR)" ]; then \ - backup_dir=$(FROZEN_MPY_DIR).$$(date +%Y%m%dT%H%M%S); mkdir $$backup_dir; \ - cd $(FROZEN_MPY_DIR); git status --ignored -u all -s . | awk ' {print $$2}' \ - | xargs --no-run-if-empty cp --parents -t ../$$backup_dir; \ - rmdir ../$$backup_dir 2>/dev/null || true; \ - git clean -d -f .; \ - fi - - if [ -n "$(FROZEN_DIR)" ]; then \ - backup_dir=$(FROZEN_DIR).$$(date +%Y%m%dT%H%M%S); mkdir $$backup_dir; \ - cd $(FROZEN_DIR); git status --ignored -u all -s . | awk ' {print $$2}' \ - | xargs --no-run-if-empty cp --parents -t ../$$backup_dir; \ - rmdir ../$$backup_dir 2>/dev/null || true; \ - git clean -d -f .; \ - fi -.PHONY: clean-frozen - print-cfg: $(ECHO) "PY_SRC = $(PY_SRC)" $(ECHO) "BUILD = $(BUILD)" diff --git a/py/py.mk b/py/py.mk index 8e1e23554f8ca..122dea225affc 100644 --- a/py/py.mk +++ b/py/py.mk @@ -227,16 +227,6 @@ ifneq ($(FROZEN_MANIFEST),) PY_O += $(BUILD)/$(BUILD)/frozen_content.o endif -# object file for frozen files -ifneq ($(FROZEN_DIR),) -PY_O += $(BUILD)/$(BUILD)/frozen.o -endif - -# object file for frozen bytecode (frozen .mpy files) -ifneq ($(FROZEN_MPY_DIR),) -PY_O += $(BUILD)/$(BUILD)/frozen_mpy.o -endif - # Sources that may contain qstrings SRC_QSTR_IGNORE = py/nlr% SRC_QSTR += $(SRC_MOD) $(filter-out $(SRC_QSTR_IGNORE),$(PY_CORE_O_BASENAME:.o=.c)) $(PY_EXTMOD_O_BASENAME:.o=.c) From 0ca194774f3ee3256a01da6e5899ac8833ce47d1 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Wed, 8 Dec 2021 16:47:33 +1100 Subject: [PATCH 4/8] tools/makemanifest.py: Merge make-frozen.py. Takes the functionality from tools/make-frozen.py, adds support for multiple frozen directories, and moves it to tools/makemanifest.py. Signed-off-by: Jim Mussared --- tools/make-frozen.py | 85 ------------------------------------------- tools/makemanifest.py | 74 +++++++++++++++++++++++++++++++++---- 2 files changed, 66 insertions(+), 93 deletions(-) delete mode 100755 tools/make-frozen.py diff --git a/tools/make-frozen.py b/tools/make-frozen.py deleted file mode 100755 index bc35d383429d6..0000000000000 --- a/tools/make-frozen.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python -# -# Create frozen modules structure for MicroPython. -# -# Usage: -# -# Have a directory with modules to be frozen (only modules, not packages -# supported so far): -# -# frozen/foo.py -# frozen/bar.py -# -# Run script, passing path to the directory above: -# -# ./make-frozen.py frozen > frozen.c -# -# Include frozen.c in your build, having defined MICROPY_MODULE_FROZEN_STR in -# config. -# -from __future__ import print_function -import sys -import os - - -def module_name(f): - return f - - -modules = [] - -if len(sys.argv) > 1: - root = sys.argv[1].rstrip("/") - root_len = len(root) - - for dirpath, dirnames, filenames in os.walk(root): - for f in filenames: - fullpath = dirpath + "/" + f - st = os.stat(fullpath) - modules.append((fullpath[root_len + 1 :], st)) - -print("#include ") -print("const char mp_frozen_str_names[] = {") -for f, st in modules: - m = module_name(f) - print('"%s\\0"' % m) -print('"\\0"};') - -print("const uint32_t mp_frozen_str_sizes[] = {") - -for f, st in modules: - print("%d," % st.st_size) - -print("0};") - -print("const char mp_frozen_str_content[] = {") -for f, st in modules: - data = open(sys.argv[1] + "/" + f, "rb").read() - - # We need to properly escape the script data to create a C string. - # When C parses hex characters of the form \x00 it keeps parsing the hex - # data until it encounters a non-hex character. Thus one must create - # strings of the form "data\x01" "abc" to properly encode this kind of - # data. We could just encode all characters as hex digits but it's nice - # to be able to read the resulting C code as ASCII when possible. - - data = bytearray(data) # so Python2 extracts each byte as an integer - esc_dict = {ord("\n"): "\\n", ord("\r"): "\\r", ord('"'): '\\"', ord("\\"): "\\\\"} - chrs = ['"'] - break_str = False - for c in data: - try: - chrs.append(esc_dict[c]) - except KeyError: - if 32 <= c <= 126: - if break_str: - chrs.append('" "') - break_str = False - chrs.append(chr(c)) - else: - chrs.append("\\x%02x" % c) - break_str = True - chrs.append('\\0"') - print("".join(chrs)) - -print('"\\0"};') diff --git a/tools/makemanifest.py b/tools/makemanifest.py index 7897a83c6ee10..e4bc4185f098a 100644 --- a/tools/makemanifest.py +++ b/tools/makemanifest.py @@ -201,8 +201,6 @@ def freeze_internal(kind, path, script, opt): if not os.path.isdir(path): raise FreezeError("freeze path must be a directory: {}".format(path)) if script is None and kind == KIND_AS_STR: - if any(f[0] == KIND_AS_STR for f in manifest_list): - raise FreezeError("can only freeze one str directory") manifest_list.append((KIND_AS_STR, path, script, opt)) elif script is None or isinstance(script, str) and script.find(".") == -1: # Recursively search `path` for files to freeze, optionally restricted @@ -235,6 +233,70 @@ def freeze_internal(kind, path, script, opt): manifest_list.append((kind, path, script, opt)) +def generate_frozen_str_content(paths): + def module_name(f): + return f + + modules = [] + output = [] + + for path in paths: + root = path.rstrip("/") + root_len = len(root) + + for dirpath, dirnames, filenames in os.walk(root): + for f in filenames: + fullpath = dirpath + "/" + f + st = os.stat(fullpath) + modules.append((path, fullpath[root_len + 1 :], st)) + + output.append("#include \n") + output.append("const char mp_frozen_str_names[] = {\n") + for _path, f, st in modules: + m = module_name(f) + output.append('"%s\\0"\n' % m) + output.append('"\\0"};\n') + + output.append("const uint32_t mp_frozen_str_sizes[] = {\n") + + for _path, f, st in modules: + output.append("%d," % st.st_size) + + output.append("0};\n") + + output.append("const char mp_frozen_str_content[] = {\n") + for path, f, st in modules: + data = open(path + "/" + f, "rb").read() + + # We need to properly escape the script data to create a C string. + # When C parses hex characters of the form \x00 it keeps parsing the hex + # data until it encounters a non-hex character. Thus one must create + # strings of the form "data\x01" "abc" to properly encode this kind of + # data. We could just encode all characters as hex digits but it's nice + # to be able to read the resulting C code as ASCII when possible. + + data = bytearray(data) # so Python2 extracts each byte as an integer + esc_dict = {ord("\n"): "\\n", ord("\r"): "\\r", ord('"'): '\\"', ord("\\"): "\\\\"} + output.append('"') + break_str = False + for c in data: + try: + output.append(esc_dict[c]) + except KeyError: + if 32 <= c <= 126: + if break_str: + output.append('" "') + break_str = False + output.append(chr(c)) + else: + output.append("\\x%02x" % c) + break_str = True + output.append('\\0"\n') + + output.append('"\\0"};\n') + return "".join(output) + + def main(): # Parse arguments import argparse @@ -264,7 +326,6 @@ def main(): sys.exit(1) # Get paths to tools - MAKE_FROZEN = VARS["MPY_DIR"] + "/tools/make-frozen.py" MPY_CROSS = VARS["MPY_DIR"] + "/mpy-cross/mpy-cross" if sys.platform == "win32": MPY_CROSS += ".exe" @@ -327,10 +388,7 @@ def main(): return # Freeze paths as strings - res, output_str = system([sys.executable, MAKE_FROZEN] + str_paths) - if res != 0: - print("error freezing strings {}: {}".format(str_paths, output_str)) - sys.exit(1) + output_str = generate_frozen_str_content(str_paths) # Freeze .mpy files if mpy_files: @@ -365,7 +423,7 @@ def main(): mkdir(args.output) with open(args.output, "wb") as f: f.write(b"//\n// Content for MICROPY_MODULE_FROZEN_STR\n//\n") - f.write(output_str) + f.write(output_str.encode()) f.write(b"//\n// Content for MICROPY_MODULE_FROZEN_MPY\n//\n") f.write(output_mpy) From 62349596fce1004941f77076960153278657a016 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Sat, 11 Dec 2021 22:40:21 +1100 Subject: [PATCH 5/8] py/frozenmod: Only search .frozen/ path. This changes makemanifest.py & mpy-tool.py to merge string and mpy names into the same list (now mp_frozen_names). The various paths for loading a frozen module(mp_find_frozen_module) and checking existence of a frozen module (mp_frozen_stat) use a common function that searches this list. In addition, the frozen lookup will now only take place if the path starts with ".frozen", which needs to be added to sys.path. Signed-off-by: Jim Mussared --- py/builtinhelp.c | 17 ++--- py/builtinimport.c | 55 ++++++++------ py/frozenmod.c | 157 +++++++++++++++++----------------------- py/frozenmod.h | 4 +- py/qstrdefs.h | 4 + shared/runtime/pyexec.c | 7 +- tools/makemanifest.py | 42 +++++------ tools/mpy-tool.py | 6 +- 8 files changed, 141 insertions(+), 151 deletions(-) diff --git a/py/builtinhelp.c b/py/builtinhelp.c index 13735635e3c90..84d69caf35bed 100644 --- a/py/builtinhelp.c +++ b/py/builtinhelp.c @@ -67,10 +67,10 @@ STATIC void mp_help_add_from_map(mp_obj_t list, const mp_map_t *map) { #if MICROPY_MODULE_FROZEN STATIC void mp_help_add_from_names(mp_obj_t list, const char *name) { while (*name) { - size_t l = strlen(name); + size_t len = strlen(name); // name should end in '.py' and we strip it off - mp_obj_list_append(list, mp_obj_new_str(name, l - 3)); - name += l + 1; + mp_obj_list_append(list, mp_obj_new_str(name, len - 3)); + name += len + 1; } } #endif @@ -80,14 +80,9 @@ STATIC void mp_help_print_modules(void) { mp_help_add_from_map(list, &mp_builtin_module_map); - #if MICROPY_MODULE_FROZEN_STR - extern const char mp_frozen_str_names[]; - mp_help_add_from_names(list, mp_frozen_str_names); - #endif - - #if MICROPY_MODULE_FROZEN_MPY - extern const char mp_frozen_mpy_names[]; - mp_help_add_from_names(list, mp_frozen_mpy_names); + #if MICROPY_MODULE_FROZEN + extern const char mp_frozen_names[]; + mp_help_add_from_names(list, mp_frozen_names); #endif // sort the list so it's printed in alphabetical order diff --git a/py/builtinimport.c b/py/builtinimport.c index 755ce779a709e..bd65d7343ab70 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -50,6 +50,9 @@ // Must be a string of one byte. #define PATH_SEP_CHAR "/" +// Virtual sys.path entry that maps to the frozen modules. +#define MP_FROZEN_PATH_PREFIX ".frozen/" + bool mp_obj_is_package(mp_obj_t module) { mp_obj_t dest[2]; mp_load_method_maybe(module, MP_QSTR___path__, dest); @@ -62,9 +65,15 @@ bool mp_obj_is_package(mp_obj_t module) { // will return whether the path is a file, directory, or doesn't exist. STATIC mp_import_stat_t stat_path_or_frozen(const char *path) { #if MICROPY_MODULE_FROZEN - mp_import_stat_t st = mp_frozen_stat(path); - if (st != MP_IMPORT_STAT_NO_EXIST) { - return st; + int frozen_type; + void *data; + // Only try and load as a frozen module if it starts with .frozen/. + const int frozen_path_prefix_len = strlen(MP_FROZEN_PATH_PREFIX); + if (strncmp(path, MP_FROZEN_PATH_PREFIX, frozen_path_prefix_len) == 0) { + mp_import_stat_t st = mp_find_frozen_module(path + frozen_path_prefix_len, &frozen_type, &data); + if (st != MP_IMPORT_STAT_NO_EXIST) { + return st; + } } #endif return mp_import_stat(path); @@ -193,32 +202,36 @@ STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code, co STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { #if MICROPY_MODULE_FROZEN || MICROPY_ENABLE_COMPILER || (MICROPY_PERSISTENT_CODE_LOAD && MICROPY_HAS_FILE_READER) - char *file_str = vstr_null_terminated_str(file); + const char *file_str = vstr_null_terminated_str(file); #endif // If we support frozen modules (either as str or mpy) then try to find the // requested filename in the list of frozen module filenames. #if MICROPY_MODULE_FROZEN void *modref; - int frozen_type = mp_find_frozen_module(file_str, file->len, &modref); - - // If we support frozen str modules and the compiler is enabled, and we - // found the filename in the list of frozen files, then load and execute it. - #if MICROPY_MODULE_FROZEN_STR - if (frozen_type == MP_FROZEN_STR) { - do_load_from_lexer(module_obj, modref); - return; - } - #endif + int frozen_type; + const int frozen_path_prefix_len = strlen(MP_FROZEN_PATH_PREFIX); + if (strncmp(file_str, MP_FROZEN_PATH_PREFIX, frozen_path_prefix_len) == 0) { + mp_find_frozen_module(file_str + frozen_path_prefix_len, &frozen_type, &modref); + + // If we support frozen str modules and the compiler is enabled, and we + // found the filename in the list of frozen files, then load and execute it. + #if MICROPY_MODULE_FROZEN_STR + if (frozen_type == MP_FROZEN_STR) { + do_load_from_lexer(module_obj, modref); + return; + } + #endif - // If we support frozen mpy modules and we found a corresponding file (and - // its data) in the list of frozen files, execute it. - #if MICROPY_MODULE_FROZEN_MPY - if (frozen_type == MP_FROZEN_MPY) { - do_execute_raw_code(module_obj, modref, file_str); - return; + // If we support frozen mpy modules and we found a corresponding file (and + // its data) in the list of frozen files, execute it. + #if MICROPY_MODULE_FROZEN_MPY + if (frozen_type == MP_FROZEN_MPY) { + do_execute_raw_code(module_obj, modref, file_str + frozen_path_prefix_len); + return; + } + #endif } - #endif #endif // MICROPY_MODULE_FROZEN diff --git a/py/frozenmod.c b/py/frozenmod.c index a250c02151e92..217b82b86a742 100644 --- a/py/frozenmod.c +++ b/py/frozenmod.c @@ -5,6 +5,7 @@ * * Copyright (c) 2015 Paul Sokolovsky * Copyright (c) 2016 Damien P. George + * Copyright (c) 2021 Jim Mussared * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -31,6 +32,13 @@ #include "py/lexer.h" #include "py/frozenmod.h" +#if MICROPY_MODULE_FROZEN + +// Null-separated frozen file names. All string-type entries are listed first, +// followed by mpy-type entries. Use mp_frozen_str_sizes to determine how +// many string entries. +extern const char mp_frozen_names[]; + #if MICROPY_MODULE_FROZEN_STR #ifndef MICROPY_MODULE_FROZEN_LEXER @@ -39,118 +47,83 @@ mp_lexer_t *MICROPY_MODULE_FROZEN_LEXER(qstr src_name, const char *str, mp_uint_t len, mp_uint_t free_len); #endif -extern const char mp_frozen_str_names[]; +// Size in bytes of each string entry, followed by a zero (terminator). extern const uint32_t mp_frozen_str_sizes[]; +// Null-separated string content. extern const char mp_frozen_str_content[]; - -// On input, *len contains size of name, on output - size of content -const char *mp_find_frozen_str(const char *str, size_t *len) { - const char *name = mp_frozen_str_names; - - size_t offset = 0; - for (int i = 0; *name != 0; i++) { - size_t l = strlen(name); - if (l == *len && !memcmp(str, name, l)) { - *len = mp_frozen_str_sizes[i]; - return mp_frozen_str_content + offset; - } - name += l + 1; - offset += mp_frozen_str_sizes[i] + 1; - } - return NULL; -} - -STATIC mp_lexer_t *mp_lexer_frozen_str(const char *str, size_t len) { - size_t name_len = len; - const char *content = mp_find_frozen_str(str, &len); - - if (content == NULL) { - return NULL; - } - - qstr source = qstr_from_strn(str, name_len); - mp_lexer_t *lex = MICROPY_MODULE_FROZEN_LEXER(source, content, len, 0); - return lex; -} - -#endif +#endif // MICROPY_MODULE_FROZEN_STR #if MICROPY_MODULE_FROZEN_MPY #include "py/emitglue.h" -extern const char mp_frozen_mpy_names[]; extern const mp_raw_code_t *const mp_frozen_mpy_content[]; -STATIC const mp_raw_code_t *mp_find_frozen_mpy(const char *str, size_t len) { - const char *name = mp_frozen_mpy_names; - for (size_t i = 0; *name != 0; i++) { - size_t l = strlen(name); - if (l == len && !memcmp(str, name, l)) { - return mp_frozen_mpy_content[i]; - } - name += l + 1; - } - return NULL; -} +#endif // MICROPY_MODULE_FROZEN_MPY -#endif +// Search for "str" as a frozen entry, returning the stat result +// (no-exist/file/dir), as well as the type (none/str/mpy) and data. +mp_import_stat_t mp_find_frozen_module(const char *str, int *frozen_type, void **data) { + size_t len = strlen(str); + const char *name = mp_frozen_names; -#if MICROPY_MODULE_FROZEN + // Count the number of str lengths we have to find how many str entries. + size_t num_str = 0; + #if MICROPY_MODULE_FROZEN_STR && MICROPY_MODULE_FROZEN_MPY + for (const uint32_t *s = mp_frozen_str_sizes; *s != 0; ++s) { + ++num_str; + } + #endif -STATIC mp_import_stat_t mp_frozen_stat_helper(const char *name, const char *str) { - size_t len = strlen(str); + for (size_t i = 0; *name != 0; i++) { + size_t entry_len = strlen(name); + if (entry_len >= len && memcmp(str, name, len) == 0) { + // Query is a prefix of the current entry. + if (entry_len == len) { + // Exact match --> file. + + #if MICROPY_MODULE_FROZEN_STR + if (i < num_str) { + *frozen_type = MP_FROZEN_STR; + // Use the size table to figure out where this index starts. + size_t offset = 0; + for (size_t j = 0; j < i; ++j) { + offset += mp_frozen_str_sizes[j] + 1; + } + size_t content_len = mp_frozen_str_sizes[i]; + const char *content = &mp_frozen_str_content[offset]; + + // Note: str & len have been updated by find_frozen_entry to strip + // the ".frozen/" prefix (to avoid this being a distinct qstr to + // the original path QSTR in frozen_content.c). + qstr source = qstr_from_strn(str, len); + mp_lexer_t *lex = MICROPY_MODULE_FROZEN_LEXER(source, content, content_len, 0); + *data = lex; + } + #endif + + #if MICROPY_MODULE_FROZEN_MPY + if (i >= num_str) { + *frozen_type = MP_FROZEN_MPY; + // Load the corresponding index as a raw_code, taking + // into account any string entries to offset by. + *data = (void *)mp_frozen_mpy_content[i - num_str]; + } + #endif - for (int i = 0; *name != 0; i++) { - size_t l = strlen(name); - if (l >= len && !memcmp(str, name, len)) { - if (name[len] == 0) { return MP_IMPORT_STAT_FILE; } else if (name[len] == '/') { + // Matches up to directory separator, this is a valid + // directory path. return MP_IMPORT_STAT_DIR; } } - name += l + 1; + // Skip null separator. + name += entry_len + 1; } - return MP_IMPORT_STAT_NO_EXIST; -} - -mp_import_stat_t mp_frozen_stat(const char *str) { - mp_import_stat_t stat; - - #if MICROPY_MODULE_FROZEN_STR - stat = mp_frozen_stat_helper(mp_frozen_str_names, str); - if (stat != MP_IMPORT_STAT_NO_EXIST) { - return stat; - } - #endif - - #if MICROPY_MODULE_FROZEN_MPY - stat = mp_frozen_stat_helper(mp_frozen_mpy_names, str); - if (stat != MP_IMPORT_STAT_NO_EXIST) { - return stat; - } - #endif + *frozen_type = MP_FROZEN_NONE; return MP_IMPORT_STAT_NO_EXIST; } -int mp_find_frozen_module(const char *str, size_t len, void **data) { - #if MICROPY_MODULE_FROZEN_STR - mp_lexer_t *lex = mp_lexer_frozen_str(str, len); - if (lex != NULL) { - *data = lex; - return MP_FROZEN_STR; - } - #endif - #if MICROPY_MODULE_FROZEN_MPY - const mp_raw_code_t *rc = mp_find_frozen_mpy(str, len); - if (rc != NULL) { - *data = (void *)rc; - return MP_FROZEN_MPY; - } - #endif - return MP_FROZEN_NONE; -} - -#endif +#endif // MICROPY_MODULE_FROZEN diff --git a/py/frozenmod.h b/py/frozenmod.h index 8a477d028e036..be735e85bdc3d 100644 --- a/py/frozenmod.h +++ b/py/frozenmod.h @@ -35,8 +35,6 @@ enum { MP_FROZEN_MPY, }; -int mp_find_frozen_module(const char *str, size_t len, void **data); -const char *mp_find_frozen_str(const char *str, size_t *len); -mp_import_stat_t mp_frozen_stat(const char *str); +mp_import_stat_t mp_find_frozen_module(const char *str, int *frozen_type, void **data); #endif // MICROPY_INCLUDED_PY_FROZENMOD_H diff --git a/py/qstrdefs.h b/py/qstrdefs.h index 5b4e0dc48e053..405813941b267 100644 --- a/py/qstrdefs.h +++ b/py/qstrdefs.h @@ -60,6 +60,10 @@ Q() Q() Q(utf-8) +#if MICROPY_MODULE_FROZEN +Q(.frozen) +#endif + #if MICROPY_ENABLE_PYSTACK Q(pystack exhausted) #endif diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index 006ec096f7bcd..89cbd5e262e12 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -674,7 +674,9 @@ int pyexec_file(const char *filename) { int pyexec_file_if_exists(const char *filename) { #if MICROPY_MODULE_FROZEN - if (mp_frozen_stat(filename) == MP_IMPORT_STAT_FILE) { + int frozen_type; + void *data; + if (mp_find_frozen_module(filename, &frozen_type, &data) == MP_IMPORT_STAT_FILE) { return pyexec_frozen_module(filename); } #endif @@ -687,7 +689,8 @@ int pyexec_file_if_exists(const char *filename) { #if MICROPY_MODULE_FROZEN int pyexec_frozen_module(const char *name) { void *frozen_data; - int frozen_type = mp_find_frozen_module(name, strlen(name), &frozen_data); + int frozen_type; + mp_find_frozen_module(name, &frozen_type, &frozen_data); switch (frozen_type) { #if MICROPY_MODULE_FROZEN_STR diff --git a/tools/makemanifest.py b/tools/makemanifest.py index e4bc4185f098a..f71e9a21e3f2b 100644 --- a/tools/makemanifest.py +++ b/tools/makemanifest.py @@ -233,12 +233,14 @@ def freeze_internal(kind, path, script, opt): manifest_list.append((kind, path, script, opt)) +# Formerly make-frozen.py. +# This generates #defines that will be use def generate_frozen_str_content(paths): def module_name(f): return f modules = [] - output = [] + output = [b"#include \n"] for path in paths: root = path.rstrip("/") @@ -250,21 +252,19 @@ def module_name(f): st = os.stat(fullpath) modules.append((path, fullpath[root_len + 1 :], st)) - output.append("#include \n") - output.append("const char mp_frozen_str_names[] = {\n") + output.append(b"#define MP_FROZEN_STR_NAMES \\\n") for _path, f, st in modules: m = module_name(f) - output.append('"%s\\0"\n' % m) - output.append('"\\0"};\n') + output.append(b'"%s\\0" \\\n' % m.encode()) + output.append(b"\n") - output.append("const uint32_t mp_frozen_str_sizes[] = {\n") + output.append(b"const uint32_t mp_frozen_str_sizes[] = { ") for _path, f, st in modules: - output.append("%d," % st.st_size) + output.append(b"%d, " % st.st_size) + output.append(b"0 };\n") - output.append("0};\n") - - output.append("const char mp_frozen_str_content[] = {\n") + output.append(b"const char mp_frozen_str_content[] = {\n") for path, f, st in modules: data = open(path + "/" + f, "rb").read() @@ -276,8 +276,8 @@ def module_name(f): # to be able to read the resulting C code as ASCII when possible. data = bytearray(data) # so Python2 extracts each byte as an integer - esc_dict = {ord("\n"): "\\n", ord("\r"): "\\r", ord('"'): '\\"', ord("\\"): "\\\\"} - output.append('"') + esc_dict = {ord("\n"): b"\\n", ord("\r"): b"\\r", ord('"'): b'\\"', ord("\\"): b"\\\\"} + output.append(b'"') break_str = False for c in data: try: @@ -285,16 +285,16 @@ def module_name(f): except KeyError: if 32 <= c <= 126: if break_str: - output.append('" "') + output.append(b'" "') break_str = False - output.append(chr(c)) + output.append(chr(c).encode()) else: - output.append("\\x%02x" % c) + output.append(b"\\x%02x" % c) break_str = True - output.append('\\0"\n') + output.append(b'\\0"\n') - output.append('"\\0"};\n') - return "".join(output) + output.append(b'"\\0"\n};\n\n') + return b"".join(output) def main(): @@ -414,8 +414,8 @@ def main(): b"const qstr_pool_t mp_qstr_frozen_const_pool = {\n" b" (qstr_pool_t*)&mp_qstr_const_pool, MP_QSTRnumber_of, 0, 0\n" b"};\n" - b'const char mp_frozen_mpy_names[1] = {"\\0"};\n' - b"const mp_raw_code_t *const mp_frozen_mpy_content[1] = {NULL};\n" + b'const char mp_frozen_names[] = { MP_FROZEN_STR_NAMES "\\0"};\n' + b"const mp_raw_code_t *const mp_frozen_mpy_content[] = {NULL};\n" ) # Generate output @@ -423,7 +423,7 @@ def main(): mkdir(args.output) with open(args.output, "wb") as f: f.write(b"//\n// Content for MICROPY_MODULE_FROZEN_STR\n//\n") - f.write(output_str.encode()) + f.write(output_str) f.write(b"//\n// Content for MICROPY_MODULE_FROZEN_MPY\n//\n") f.write(output_mpy) diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 6868ed5d4e429..aa0272111c406 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -886,7 +886,11 @@ def freeze_mpy(base_qstrs, raw_codes): rc.freeze(rc.source_file.str.replace("/", "_")[:-3] + "_") print() - print("const char mp_frozen_mpy_names[] = {") + print("const char mp_frozen_names[] = {") + print("#ifdef MP_FROZEN_STR_NAMES") + # makemanifest.py might also include some frozen string content. + print("MP_FROZEN_STR_NAMES") + print("#endif") for rc in raw_codes: module_name = rc.source_file.str print('"%s\\0"' % module_name) From f7b3751ae61e768e3eae355eef2bcfe50e4dc357 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Sat, 11 Dec 2021 23:01:58 +1100 Subject: [PATCH 6/8] py/mkrules.cmake: Set frozen defs early. This ensures MICROPY_QSTR_EXTRA_POOL and MICROPY_MODULE_FROZEN_MPY are set if necessary before the CFLAGS are extracted for QSTR generation. Signed-off-by: Jim Mussared --- py/mkrules.cmake | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/py/mkrules.cmake b/py/mkrules.cmake index 9d08017931963..cb5fdabf6b933 100644 --- a/py/mkrules.cmake +++ b/py/mkrules.cmake @@ -10,6 +10,15 @@ set(MICROPY_QSTRDEFS_COLLECTED "${MICROPY_GENHDR_DIR}/qstrdefs.collected.h") set(MICROPY_QSTRDEFS_PREPROCESSED "${MICROPY_GENHDR_DIR}/qstrdefs.preprocessed.h") set(MICROPY_QSTRDEFS_GENERATED "${MICROPY_GENHDR_DIR}/qstrdefs.generated.h") +# Need to do this before extracting MICROPY_CPP_DEF below. Rest of frozen +# manifest handling is at the end of this file. +if(MICROPY_FROZEN_MANIFEST) + target_compile_definitions(${MICROPY_TARGET} PUBLIC + MICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool + MICROPY_MODULE_FROZEN_MPY=\(1\) + ) +endif() + # Provide defaults for preprocessor flags if not already defined if(NOT MICROPY_CPP_FLAGS) get_target_property(MICROPY_CPP_INC ${MICROPY_TARGET} INCLUDE_DIRECTORIES) @@ -120,10 +129,7 @@ if(MICROPY_FROZEN_MANIFEST) ${MICROPY_FROZEN_CONTENT} ) - target_compile_definitions(${MICROPY_TARGET} PUBLIC - MICROPY_QSTR_EXTRA_POOL=mp_qstr_frozen_const_pool - MICROPY_MODULE_FROZEN_MPY=\(1\) - ) + # Note: target_compile_definitions already added earlier. if(NOT MICROPY_LIB_DIR) set(MICROPY_LIB_DIR ${MICROPY_DIR}/../micropython-lib) From d833f02eebbd0df80b84c438face4d937d255ae7 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Sat, 11 Dec 2021 23:04:01 +1100 Subject: [PATCH 7/8] ports: Add .frozen as first entry in sys.path. Frozen modules will be searched preferentially, but gives the user the ability to override this behavior. This matches the previous behavior where "" was implicitly the frozen search path, but the frozen list was checked before the filesystem. Signed-off-by: Jim Mussared --- ports/esp32/main.c | 3 +++ ports/esp8266/main.c | 3 +++ ports/mimxrt/main.c | 3 +++ ports/nrf/main.c | 3 +++ ports/rp2/main.c | 3 +++ ports/stm32/main.c | 3 +++ ports/unix/main.c | 13 +++++-------- ports/unix/mpconfigport.h | 3 +++ ports/unix/variants/minimal/mpconfigvariant.h | 3 +++ shared/upytesthelper/upytesthelper.c | 3 +++ 10 files changed, 32 insertions(+), 8 deletions(-) diff --git a/ports/esp32/main.c b/ports/esp32/main.c index 4920180b23512..e95930ed071dd 100644 --- a/ports/esp32/main.c +++ b/ports/esp32/main.c @@ -141,6 +141,9 @@ void mp_task(void *pvParameter) { gc_init(mp_task_heap, mp_task_heap + mp_task_heap_size); mp_init(); mp_obj_list_init(mp_sys_path, 0); + #if MICROPY_MODULE_FROZEN + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__dot_frozen)); + #endif mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_lib)); mp_obj_list_init(mp_sys_argv, 0); diff --git a/ports/esp8266/main.c b/ports/esp8266/main.c index 404188346cc00..14863a35b0a74 100644 --- a/ports/esp8266/main.c +++ b/ports/esp8266/main.c @@ -53,6 +53,9 @@ STATIC void mp_reset(void) { gc_init(heap, heap + sizeof(heap)); mp_init(); mp_obj_list_init(mp_sys_path, 0); + #if MICROPY_MODULE_FROZEN + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__dot_frozen)); + #endif mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); // current dir (or base dir of the script) mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_lib)); mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_)); diff --git a/ports/mimxrt/main.c b/ports/mimxrt/main.c index 4d603913fdf01..28ee30d990b27 100644 --- a/ports/mimxrt/main.c +++ b/ports/mimxrt/main.c @@ -76,6 +76,9 @@ int main(void) { mp_init(); mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_path), 0); + #if MICROPY_MODULE_FROZEN + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__dot_frozen)); + #endif mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_argv), 0); #if MICROPY_PY_NETWORK diff --git a/ports/nrf/main.c b/ports/nrf/main.c index 3768b8f404862..fc67bd657db6f 100644 --- a/ports/nrf/main.c +++ b/ports/nrf/main.c @@ -130,6 +130,9 @@ int main(int argc, char **argv) { mp_init(); mp_obj_list_init(mp_sys_path, 0); + #if MICROPY_MODULE_FROZEN + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__dot_frozen)); + #endif mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); // current dir (or base dir of the script) mp_obj_list_init(mp_sys_argv, 0); diff --git a/ports/rp2/main.c b/ports/rp2/main.c index 793444c9a92cf..6a5bb8867dc7a 100644 --- a/ports/rp2/main.c +++ b/ports/rp2/main.c @@ -101,6 +101,9 @@ int main(int argc, char **argv) { // Initialise MicroPython runtime. mp_init(); mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_path), 0); + #if MICROPY_MODULE_FROZEN + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__dot_frozen)); + #endif mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_lib)); mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_argv), 0); diff --git a/ports/stm32/main.c b/ports/stm32/main.c index 974602cb05f16..d85843d422803 100644 --- a/ports/stm32/main.c +++ b/ports/stm32/main.c @@ -495,6 +495,9 @@ void stm32_main(uint32_t reset_mode) { // MicroPython init mp_init(); mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_path), 0); + #if MICROPY_MODULE_FROZEN + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__dot_frozen)); + #endif mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR_)); // current dir (or base dir of the script) mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_argv), 0); diff --git a/ports/unix/main.c b/ports/unix/main.c index 031bdd75d106e..fd4fd1c0f819e 100644 --- a/ports/unix/main.c +++ b/ports/unix/main.c @@ -495,13 +495,9 @@ MP_NOINLINE int main_(int argc, char **argv) { char *home = getenv("HOME"); char *path = getenv("MICROPYPATH"); if (path == NULL) { - #ifdef MICROPY_PY_SYS_PATH_DEFAULT path = MICROPY_PY_SYS_PATH_DEFAULT; - #else - path = "~/.micropython/lib:/usr/lib/micropython"; - #endif } - size_t path_num = 1; // [0] is for current dir (or base dir of the script) + size_t path_num = 2; // [0] is frozen, [1] is for current dir (or base dir of the script) if (*path == PATHLIST_SEP_CHAR) { path_num++; } @@ -514,10 +510,11 @@ MP_NOINLINE int main_(int argc, char **argv) { mp_obj_list_init(MP_OBJ_TO_PTR(mp_sys_path), path_num); mp_obj_t *path_items; mp_obj_list_get(mp_sys_path, &path_num, &path_items); - path_items[0] = MP_OBJ_NEW_QSTR(MP_QSTR_); + path_items[0] = MP_OBJ_NEW_QSTR(MP_QSTR__dot_frozen); + path_items[1] = MP_OBJ_NEW_QSTR(MP_QSTR_); { char *p = path; - for (mp_uint_t i = 1; i < path_num; i++) { + for (mp_uint_t i = 2; i < path_num; i++) { char *p1 = strchr(p, PATHLIST_SEP_CHAR); if (p1 == NULL) { p1 = p + strlen(p); @@ -660,7 +657,7 @@ MP_NOINLINE int main_(int argc, char **argv) { // Set base dir of the script as first entry in sys.path char *p = strrchr(basedir, '/'); - path_items[0] = mp_obj_new_str_via_qstr(basedir, p - basedir); + path_items[1] = mp_obj_new_str_via_qstr(basedir, p - basedir); free(pathbuf); set_sys_argv(argv, argc, a); diff --git a/ports/unix/mpconfigport.h b/ports/unix/mpconfigport.h index 93d790ac36f1e..68d4d8d33fe5e 100644 --- a/ports/unix/mpconfigport.h +++ b/ports/unix/mpconfigport.h @@ -123,6 +123,9 @@ #define MICROPY_PY_SYS_PLATFORM "linux" #endif #endif +#ifndef MICROPY_PY_SYS_PATH_DEFAULT +#define MICROPY_PY_SYS_PATH_DEFAULT "~/.micropython/lib:/usr/lib/micropython" +#endif #define MICROPY_PY_SYS_MAXSIZE (1) #define MICROPY_PY_SYS_STDFILES (1) #define MICROPY_PY_SYS_EXC_INFO (1) diff --git a/ports/unix/variants/minimal/mpconfigvariant.h b/ports/unix/variants/minimal/mpconfigvariant.h index 973febd572637..4766b8e8953aa 100644 --- a/ports/unix/variants/minimal/mpconfigvariant.h +++ b/ports/unix/variants/minimal/mpconfigvariant.h @@ -88,6 +88,9 @@ #define MICROPY_PY_SYS (1) #define MICROPY_PY_SYS_EXIT (0) #define MICROPY_PY_SYS_PLATFORM "linux" +#ifndef MICROPY_PY_SYS_PATH_DEFAULT +#define MICROPY_PY_SYS_PATH_DEFAULT "~/.micropython/lib:/usr/lib/micropython" +#endif #define MICROPY_PY_SYS_MAXSIZE (0) #define MICROPY_PY_SYS_STDFILES (0) #define MICROPY_PY_CMATH (0) diff --git a/shared/upytesthelper/upytesthelper.c b/shared/upytesthelper/upytesthelper.c index 326172be658c0..ce6073242438e 100644 --- a/shared/upytesthelper/upytesthelper.c +++ b/shared/upytesthelper/upytesthelper.c @@ -94,6 +94,9 @@ void upytest_execute_test(const char *src) { gc_init(heap_start, heap_end); mp_init(); mp_obj_list_init(mp_sys_path, 0); + #if MICROPY_MODULE_FROZEN + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__dot_frozen)); + #endif mp_obj_list_init(mp_sys_argv, 0); nlr_buf_t nlr; From f1470c38a38de977056c22683cb6ecd068cc5991 Mon Sep 17 00:00:00 2001 From: Jim Mussared Date: Sat, 11 Dec 2021 23:39:40 +1100 Subject: [PATCH 8/8] docs/library/sys.rst: Add note about .frozen. Signed-off-by: Jim Mussared --- docs/library/sys.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/library/sys.rst b/docs/library/sys.rst index 80ac1ab84c0df..e675c66d20f0f 100644 --- a/docs/library/sys.rst +++ b/docs/library/sys.rst @@ -115,6 +115,11 @@ Constants A mutable list of directories to search for imported modules. + .. admonition:: Difference to CPython + :class: attention + + On MicroPython, an entry with the value ``".frozen"`` will indicate that it should search :term:`frozen modules `. + .. data:: platform The platform that MicroPython is running on. For OS/RTOS ports, this is