diff --git a/docs/reference/mpremote.rst b/docs/reference/mpremote.rst index d4c6983595fec..ef23cd85c21a3 100644 --- a/docs/reference/mpremote.rst +++ b/docs/reference/mpremote.rst @@ -78,6 +78,7 @@ The full list of supported commands are: - `mip ` - `mount ` - `unmount ` +- `romfs ` - `rtc ` - `sleep ` - `reset ` @@ -347,6 +348,29 @@ The full list of supported commands are: This happens automatically when ``mpremote`` terminates, but it can be used in a sequence to unmount an earlier mount before subsequent command are run. +.. _mpremote_command_romfs: + +- **romfs** -- manage ROMFS partitions on the device: + + .. code-block:: bash + + $ mpremote romfs + + ```` may be: + + - ``romfs query`` to list all the available ROMFS partitions and their size + - ``romfs [-o ] build `` to create a ROMFS image from the given + source directory; the default output file is the source appended by ``.romfs`` + - ``romfs [-p ] deploy `` to deploy a ROMFS image to the device; + will also create a temporary ROMFS image if the source is a directory + + The ``build`` and ``deploy`` sub-commands both support the ``-m``/``--mpy`` option + to automatically compile ``.py`` files to ``.mpy`` when creating the ROMFS image. + This option is enabled by default, but only works if the ``mpy_cross`` Python + package has been installed (eg via ``pip install mpy_cross``). If the package is + not installed then a warning is printed and ``.py`` files remain as is. Compiling + of ``.py`` files can be disabled with the ``--no-mpy`` option. + .. _mpremote_command_rtc: - **rtc** -- set/get the device clock (RTC): diff --git a/extmod/modvfs.c b/extmod/modvfs.c index df422365b75ef..41841f05569a2 100644 --- a/extmod/modvfs.c +++ b/extmod/modvfs.c @@ -38,11 +38,18 @@ #error "MICROPY_PY_VFS requires MICROPY_VFS" #endif +#if MICROPY_VFS_ROM_IOCTL +static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_vfs_rom_ioctl_obj, 1, 4, mp_vfs_rom_ioctl); +#endif + static const mp_rom_map_elem_t vfs_module_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_vfs) }, { MP_ROM_QSTR(MP_QSTR_mount), MP_ROM_PTR(&mp_vfs_mount_obj) }, { MP_ROM_QSTR(MP_QSTR_umount), MP_ROM_PTR(&mp_vfs_umount_obj) }, + #if MICROPY_VFS_ROM_IOCTL + { MP_ROM_QSTR(MP_QSTR_rom_ioctl), MP_ROM_PTR(&mp_vfs_rom_ioctl_obj) }, + #endif #if MICROPY_VFS_FAT { MP_ROM_QSTR(MP_QSTR_VfsFat), MP_ROM_PTR(&mp_fat_vfs_type) }, #endif diff --git a/extmod/vfs.c b/extmod/vfs.c index b9c5ab0fc3a8c..c83b8ce7d563e 100644 --- a/extmod/vfs.c +++ b/extmod/vfs.c @@ -46,6 +46,10 @@ #include "extmod/vfs_posix.h" #endif +#if MICROPY_VFS_ROM && MICROPY_VFS_ROM_IOCTL +#include "extmod/vfs_rom.h" +#endif + // For mp_vfs_proxy_call, the maximum number of additional args that can be passed. // A fixed maximum size is used to avoid the need for a costly variable array. #define PROXY_MAX_ARGS (2) @@ -552,6 +556,32 @@ int mp_vfs_mount_and_chdir_protected(mp_obj_t bdev, mp_obj_t mount_point) { return ret; } +#if MICROPY_VFS_ROM && MICROPY_VFS_ROM_IOCTL + +int mp_vfs_mount_romfs_protected(void) { + int ret; + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_obj_t args[2] = { MP_OBJ_NEW_SMALL_INT(MP_VFS_ROM_IOCTL_GET_SEGMENT), MP_OBJ_NEW_SMALL_INT(0) }; + mp_obj_t rom = mp_vfs_rom_ioctl(2, args); + mp_obj_t romfs = mp_call_function_1(MP_OBJ_FROM_PTR(&mp_type_vfs_rom), rom); + mp_obj_t mount_point = MP_OBJ_NEW_QSTR(MP_QSTR__slash_rom); + mp_call_function_2(MP_OBJ_FROM_PTR(&mp_vfs_mount_obj), romfs, mount_point); + #if MICROPY_PY_SYS_PATH_ARGV_DEFAULTS + // Add "/rom" and "/rom/lib" to `sys.path`. + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_rom)); + mp_obj_list_append(mp_sys_path, MP_OBJ_NEW_QSTR(MP_QSTR__slash_rom_slash_lib)); + #endif + ret = 0; // success + nlr_pop(); + } else { + ret = -MP_EIO; + } + return ret; +} + +#endif + MP_REGISTER_ROOT_POINTER(struct _mp_vfs_mount_t *vfs_cur); MP_REGISTER_ROOT_POINTER(struct _mp_vfs_mount_t *vfs_mount_table); diff --git a/extmod/vfs.h b/extmod/vfs.h index 626e25a351123..3acc09db12283 100644 --- a/extmod/vfs.h +++ b/extmod/vfs.h @@ -52,6 +52,13 @@ #define MP_BLOCKDEV_IOCTL_BLOCK_SIZE (5) #define MP_BLOCKDEV_IOCTL_BLOCK_ERASE (6) +// Constants for vfs.rom_ioctl() function. +#define MP_VFS_ROM_IOCTL_GET_NUMBER_OF_SEGMENTS (1) // rom_ioctl(1) +#define MP_VFS_ROM_IOCTL_GET_SEGMENT (2) // rom_ioctl(2, ) +#define MP_VFS_ROM_IOCTL_WRITE_PREPARE (3) // rom_ioctl(3, , ) +#define MP_VFS_ROM_IOCTL_WRITE (4) // rom_ioctl(4, , , ) +#define MP_VFS_ROM_IOCTL_WRITE_COMPLETE (5) // rom_ioctl(5, ) + // At the moment the VFS protocol just has import_stat, but could be extended to other methods typedef struct _mp_vfs_proto_t { mp_import_stat_t (*import_stat)(void *self, const char *path); @@ -105,6 +112,9 @@ mp_obj_t mp_vfs_stat(mp_obj_t path_in); mp_obj_t mp_vfs_statvfs(mp_obj_t path_in); int mp_vfs_mount_and_chdir_protected(mp_obj_t bdev, mp_obj_t mount_point); +#if MICROPY_VFS_ROM && MICROPY_VFS_ROM_IOCTL +int mp_vfs_mount_romfs_protected(void); +#endif MP_DECLARE_CONST_FUN_OBJ_KW(mp_vfs_mount_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_vfs_umount_obj); @@ -122,4 +132,12 @@ MP_DECLARE_CONST_FUN_OBJ_1(mp_vfs_rmdir_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_vfs_stat_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_vfs_statvfs_obj); +#if MICROPY_VFS_ROM_IOCTL +// When MICROPY_VFS_ROM_IOCTL is enabled a port must define the following function. +// This is a generic interface to allow querying and modifying the user-accessible, +// read-only memory area of a device, if it is configured with such an area. +// Supported ioctl commands are given by MP_VFS_ROM_IOCTL_xxx. +mp_obj_t mp_vfs_rom_ioctl(size_t n_args, const mp_obj_t *args); +#endif + #endif // MICROPY_INCLUDED_EXTMOD_VFS_H diff --git a/ports/esp32/esp32_partition.c b/ports/esp32/esp32_partition.c index 55830a285b0b6..f33e9da671b8c 100644 --- a/ports/esp32/esp32_partition.c +++ b/ports/esp32/esp32_partition.c @@ -53,6 +53,20 @@ typedef struct _esp32_partition_obj_t { uint16_t block_size; } esp32_partition_obj_t; +#if MICROPY_VFS_ROM_IOCTL + +static esp32_partition_obj_t esp32_partition_romfs_obj = { + .base = { .type = NULL }, + .part = NULL, + .cache = NULL, + .block_size = NATIVE_BLOCK_SIZE_BYTES, +}; + +static const void *esp32_partition_romfs_ptr = NULL; +static esp_partition_mmap_handle_t esp32_partition_romfs_handle; + +#endif + static esp32_partition_obj_t *esp32_partition_new(const esp_partition_t *part, uint16_t block_size) { if (part == NULL) { mp_raise_OSError(MP_ENOENT); @@ -114,6 +128,24 @@ static mp_obj_t esp32_partition_make_new(const mp_obj_type_t *type, size_t n_arg return MP_OBJ_FROM_PTR(esp32_partition_new(part, block_size)); } +#if MICROPY_VFS_ROM_IOCTL +static mp_int_t esp32_partition_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, mp_uint_t flags) { + esp32_partition_obj_t *self = MP_OBJ_TO_PTR(self_in); + if (self == &esp32_partition_romfs_obj && flags == MP_BUFFER_READ) { + if (esp32_partition_romfs_ptr == NULL) { + check_esp_err(esp_partition_mmap(self->part, 0, self->part->size, ESP_PARTITION_MMAP_DATA, &esp32_partition_romfs_ptr, &esp32_partition_romfs_handle)); + } + bufinfo->buf = (void *)esp32_partition_romfs_ptr; + bufinfo->len = self->part->size; + bufinfo->typecode = 'B'; + return 0; + } else { + // Unsupported. + return 1; + } +} +#endif + static mp_obj_t esp32_partition_find(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { // Parse args enum { ARG_type, ARG_subtype, ARG_label, ARG_block_size }; @@ -284,11 +316,55 @@ static const mp_rom_map_elem_t esp32_partition_locals_dict_table[] = { }; static MP_DEFINE_CONST_DICT(esp32_partition_locals_dict, esp32_partition_locals_dict_table); +#if MICROPY_VFS_ROM_IOCTL +#define ESP32_PARTITION_GET_BUFFER buffer, esp32_partition_get_buffer, +#else +#define ESP32_PARTITION_GET_BUFFER +#endif + MP_DEFINE_CONST_OBJ_TYPE( esp32_partition_type, MP_QSTR_Partition, MP_TYPE_FLAG_NONE, make_new, esp32_partition_make_new, print, esp32_partition_print, + ESP32_PARTITION_GET_BUFFER locals_dict, &esp32_partition_locals_dict ); + +/******************************************************************************/ +// romfs partition + +#if MICROPY_VFS_ROM_IOCTL + +mp_obj_t mp_vfs_rom_ioctl(size_t n_args, const mp_obj_t *args) { + if (esp32_partition_romfs_obj.base.type == NULL) { + esp32_partition_romfs_obj.base.type = &esp32_partition_type; + // Get the romfs partition. + // TODO: number of segments ioctl can be used if there is more than one romfs. + esp_partition_iterator_t iter = esp_partition_find(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_ANY, "romfs"); + if (iter != NULL) { + esp32_partition_romfs_obj.part = esp_partition_get(iter); + } + esp_partition_iterator_release(iter); + } + + switch (mp_obj_get_int(args[0])) { + case MP_VFS_ROM_IOCTL_GET_NUMBER_OF_SEGMENTS: + if (esp32_partition_romfs_obj.part == NULL) { + return MP_OBJ_NEW_SMALL_INT(0); + } else { + return MP_OBJ_NEW_SMALL_INT(1); + } + case MP_VFS_ROM_IOCTL_GET_SEGMENT: + if (esp32_partition_romfs_obj.part == NULL) { + return MP_OBJ_NEW_SMALL_INT(-MP_EINVAL); + } else { + return MP_OBJ_FROM_PTR(&esp32_partition_romfs_obj); + } + default: + return MP_OBJ_NEW_SMALL_INT(-MP_EINVAL); + } +} + +#endif // MICROPY_VFS_ROM_IOCTL diff --git a/ports/esp32/partitions-4MiB-romfs.csv b/ports/esp32/partitions-4MiB-romfs.csv new file mode 100644 index 0000000000000..dd02506e54684 --- /dev/null +++ b/ports/esp32/partitions-4MiB-romfs.csv @@ -0,0 +1,8 @@ +# Notes: the offset of the partition table itself is set in +# $IDF_PATH/components/partition_table/Kconfig.projbuild. +# Name, Type, SubType, Offset, Size, Flags +nvs, data, nvs, 0x9000, 0x6000, +phy_init, data, phy, 0xf000, 0x1000, +factory, app, factory, 0x10000, 0x1D0000, +romfs, data, 0x8f, 0x1E0000, 0x20000, +vfs, data, fat, 0x200000, 0x200000, diff --git a/ports/esp8266/Makefile b/ports/esp8266/Makefile index 1721075eae172..1c9de01503abd 100644 --- a/ports/esp8266/Makefile +++ b/ports/esp8266/Makefile @@ -134,6 +134,7 @@ SRC_C = \ fatfs_port.c \ posix_helpers.c \ hspi.c \ + vfs_rom_ioctl.c \ $(wildcard $(BOARD_DIR)/*.c) \ ifeq ($(MICROPY_PY_ESPNOW),1) diff --git a/ports/esp8266/boards/ESP8266_GENERIC/board.json b/ports/esp8266/boards/ESP8266_GENERIC/board.json index 2b3a6b5507db8..8f2aa1240f280 100644 --- a/ports/esp8266/boards/ESP8266_GENERIC/board.json +++ b/ports/esp8266/boards/ESP8266_GENERIC/board.json @@ -17,7 +17,8 @@ "variants": { "OTA": "OTA compatible", "FLASH_1M": "1MiB flash", - "FLASH_512K": "512kiB flash" + "FLASH_512K": "512kiB flash", + "FLASH_2M_ROMFS": "2MiB flash with ROMFS" }, "vendor": "Espressif" } diff --git a/ports/esp8266/boards/ESP8266_GENERIC/mpconfigvariant_FLASH_2M_ROMFS.mk b/ports/esp8266/boards/ESP8266_GENERIC/mpconfigvariant_FLASH_2M_ROMFS.mk new file mode 100644 index 0000000000000..9ee3566d8bf35 --- /dev/null +++ b/ports/esp8266/boards/ESP8266_GENERIC/mpconfigvariant_FLASH_2M_ROMFS.mk @@ -0,0 +1,12 @@ +LD_FILES = boards/esp8266_2MiB_ROMFS.ld + +MICROPY_PY_ESPNOW ?= 1 +MICROPY_PY_BTREE ?= 1 +MICROPY_VFS_FAT ?= 1 +MICROPY_VFS_LFS2 ?= 1 + +# Add asyncio and extra micropython-lib packages (in addition to the port manifest). +FROZEN_MANIFEST ?= $(BOARD_DIR)/manifest_2MiB.py + +# Configure mpconfigboard.h. +CFLAGS += -DMICROPY_ESP8266_2M -DMICROPY_VFS_ROM=1 diff --git a/ports/esp8266/boards/esp8266_2MiB_ROMFS.ld b/ports/esp8266/boards/esp8266_2MiB_ROMFS.ld new file mode 100644 index 0000000000000..6020c3d485da6 --- /dev/null +++ b/ports/esp8266/boards/esp8266_2MiB_ROMFS.ld @@ -0,0 +1,24 @@ +/* GNU linker script for ESP8266 with 2M or more flash, and includes a ROMFS partition + + Flash layout: + 0x40200000 36k header + iram/dram init + 0x40209000 668k firmware (irom0) + 0x402c0000 320k ROMFS + 0x40300000 1M+ filesystem (not memory mapped) +*/ + +MEMORY +{ + dport0_0_seg : org = 0x3ff00000, len = 16 + dram0_0_seg : org = 0x3ffe8000, len = 80K + iram1_0_seg : org = 0x40100000, len = 32K + irom0_0_seg : org = 0x40209000, len = 1M - 36K - 320K + FLASH_ROMFS : org = 0x402b0000, len = 320K +} + +/* define ROMFS extents */ +_micropy_hw_romfs_start = ORIGIN(FLASH_ROMFS); +_micropy_hw_romfs_size = LENGTH(FLASH_ROMFS); + +/* define common sections and symbols */ +INCLUDE boards/esp8266_common.ld diff --git a/ports/esp8266/boards/esp8266_common.ld b/ports/esp8266/boards/esp8266_common.ld index cf4883acc9d20..206c3f14b430e 100644 --- a/ports/esp8266/boards/esp8266_common.ld +++ b/ports/esp8266/boards/esp8266_common.ld @@ -170,6 +170,7 @@ SECTIONS *modonewire.o(.literal* .text*) *network_wlan.o(.literal* .text*) *esp_mphal.o(.literal* .text*) + *vfs_rom_ioctl.o(.literal* .text*) /* we put as much rodata as possible in this section */ /* note that only rodata accessed as a machine word is allowed here */ diff --git a/ports/esp8266/modesp.c b/ports/esp8266/modesp.c index f303da1a33289..c88022945cfe6 100644 --- a/ports/esp8266/modesp.c +++ b/ports/esp8266/modesp.c @@ -164,9 +164,16 @@ static MP_DEFINE_CONST_FUN_OBJ_0(esp_flash_size_obj, esp_flash_size); #define IS_OTA_FIRMWARE() ((*(uint32_t *)0x40200000 & 0xff00) == 0x100) extern byte _firmware_size[]; +#if MICROPY_VFS_ROM_IOCTL +extern uint8_t _micropy_hw_romfs_size; +#endif static mp_obj_t esp_flash_user_start(void) { - return MP_OBJ_NEW_SMALL_INT((uint32_t)_firmware_size); + uint32_t flash_user_start = (uint32_t)_firmware_size; + #if MICROPY_VFS_ROM_IOCTL + flash_user_start += (uint32_t)&_micropy_hw_romfs_size; + #endif + return MP_OBJ_NEW_SMALL_INT(flash_user_start); } static MP_DEFINE_CONST_FUN_OBJ_0(esp_flash_user_start_obj, esp_flash_user_start); diff --git a/ports/esp8266/vfs_rom_ioctl.c b/ports/esp8266/vfs_rom_ioctl.c new file mode 100644 index 0000000000000..c7528375becd0 --- /dev/null +++ b/ports/esp8266/vfs_rom_ioctl.c @@ -0,0 +1,89 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2024-2025 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/mperrno.h" +#include "py/objarray.h" +#include "extmod/vfs.h" +#include "ets_alt_task.h" +#include "user_interface.h" + +#if MICROPY_VFS_ROM_IOCTL + +#define FLASH_MEM_BASE (0x40200000) +#define FLASH_PAGE_SIZE (4096) + +#define MICROPY_HW_ROMFS_BASE (uintptr_t)(&_micropy_hw_romfs_start) +#define MICROPY_HW_ROMFS_BYTES (uintptr_t)(&_micropy_hw_romfs_size) + +#define ROMFS_SPI_FLASH_OFFSET (MICROPY_HW_ROMFS_BASE - FLASH_MEM_BASE) + +extern uint8_t _micropy_hw_romfs_start; +extern uint8_t _micropy_hw_romfs_size; + +static const MP_DEFINE_MEMORYVIEW_OBJ(romfs_obj, 'B', 0, MICROPY_HW_ROMFS_BYTES, (void *)MICROPY_HW_ROMFS_BASE); + +mp_obj_t mp_vfs_rom_ioctl(size_t n_args, const mp_obj_t *args) { + switch (mp_obj_get_int(args[0])) { + case MP_VFS_ROM_IOCTL_GET_NUMBER_OF_SEGMENTS: + return MP_OBJ_NEW_SMALL_INT(1); + + case MP_VFS_ROM_IOCTL_GET_SEGMENT: + return MP_OBJ_FROM_PTR(&romfs_obj); + + case MP_VFS_ROM_IOCTL_WRITE_PREPARE: { + uint32_t dest = ROMFS_SPI_FLASH_OFFSET; + uint32_t dest_max = dest + mp_obj_get_int(args[2]); + while (dest < dest_max) { + ets_loop_iter(); // flash access takes time so run any pending tasks + SpiFlashOpResult res = spi_flash_erase_sector(dest / FLASH_PAGE_SIZE); + ets_loop_iter(); + if (res != SPI_FLASH_RESULT_OK) { + return MP_OBJ_NEW_SMALL_INT(res == SPI_FLASH_RESULT_TIMEOUT ? -MP_ETIMEDOUT : -MP_EIO); + } + dest += FLASH_PAGE_SIZE; + } + return MP_OBJ_NEW_SMALL_INT(4); // minimum write size + } + + case MP_VFS_ROM_IOCTL_WRITE: { + mp_int_t offset = ROMFS_SPI_FLASH_OFFSET + mp_obj_get_int(args[2]); + mp_buffer_info_t bufinfo; + mp_get_buffer_raise(args[3], &bufinfo, MP_BUFFER_READ); + ets_loop_iter(); // flash access takes time so run any pending tasks + SpiFlashOpResult res = spi_flash_write(offset, bufinfo.buf, (bufinfo.len + 3) & ~3); + ets_loop_iter(); + if (res == SPI_FLASH_RESULT_OK) { + return MP_OBJ_NEW_SMALL_INT(0); + } + return MP_OBJ_NEW_SMALL_INT(res == SPI_FLASH_RESULT_TIMEOUT ? -MP_ETIMEDOUT : -MP_EIO); + } + + default: + return MP_OBJ_NEW_SMALL_INT(-MP_EINVAL); + } +} + +#endif // MICROPY_VFS_ROM_IOCTL diff --git a/ports/qemu/mpconfigport.h b/ports/qemu/mpconfigport.h index 0a25df4d0ac10..b02507277323e 100644 --- a/ports/qemu/mpconfigport.h +++ b/ports/qemu/mpconfigport.h @@ -64,6 +64,7 @@ #define MICROPY_PY_MACHINE_PIN_BASE (1) #define MICROPY_VFS (1) #define MICROPY_VFS_ROM (1) +#define MICROPY_VFS_ROM_IOCTL (0) // type definitions for the specific machine diff --git a/ports/rp2/mpconfigport.h b/ports/rp2/mpconfigport.h index fe8287ba1d6e5..38dcad1dae6e1 100644 --- a/ports/rp2/mpconfigport.h +++ b/ports/rp2/mpconfigport.h @@ -67,6 +67,16 @@ #endif #endif +// Number of bytes of flash to allocate to the ROMFS partition. +#ifndef MICROPY_HW_ROMFS_BYTES +#define MICROPY_HW_ROMFS_BYTES (0) +#endif + +// Number of bytes of flash to allocate to read/write filesystem storage. +#ifndef MICROPY_HW_FLASH_STORAGE_BYTES +#define MICROPY_HW_FLASH_STORAGE_BYTES (1408 * 1024) +#endif + #ifndef MICROPY_CONFIG_ROM_LEVEL #define MICROPY_CONFIG_ROM_LEVEL (MICROPY_CONFIG_ROM_LEVEL_EXTRA_FEATURES) #endif @@ -170,6 +180,7 @@ #define MICROPY_VFS (1) #define MICROPY_VFS_LFS2 (1) #define MICROPY_VFS_FAT (1) +#define MICROPY_VFS_ROM (MICROPY_HW_ROMFS_BYTES > 0) #define MICROPY_SSL_MBEDTLS (1) #define MICROPY_PY_LWIP_PPP (MICROPY_PY_NETWORK_PPP_LWIP) #define MICROPY_PY_LWIP_SOCK_RAW (MICROPY_PY_LWIP) diff --git a/ports/rp2/rp2_flash.c b/ports/rp2/rp2_flash.c index a487fb1633b21..f3f154a1402d9 100644 --- a/ports/rp2/rp2_flash.c +++ b/ports/rp2/rp2_flash.c @@ -28,6 +28,7 @@ #include "py/mphal.h" #include "py/runtime.h" +#include "py/mperrno.h" #include "extmod/vfs.h" #include "modrp2.h" #include "hardware/flash.h" @@ -35,15 +36,16 @@ #define BLOCK_SIZE_BYTES (FLASH_SECTOR_SIZE) -#ifndef MICROPY_HW_FLASH_STORAGE_BYTES -#define MICROPY_HW_FLASH_STORAGE_BYTES (1408 * 1024) -#endif +static_assert(MICROPY_HW_ROMFS_BYTES % 4096 == 0, "ROMFS size must be a multiple of 4K"); static_assert(MICROPY_HW_FLASH_STORAGE_BYTES % 4096 == 0, "Flash storage size must be a multiple of 4K"); #ifndef MICROPY_HW_FLASH_STORAGE_BASE #define MICROPY_HW_FLASH_STORAGE_BASE (PICO_FLASH_SIZE_BYTES - MICROPY_HW_FLASH_STORAGE_BYTES) #endif +// Put ROMFS at the upper end of the code space. +#define MICROPY_HW_ROMFS_BASE (MICROPY_HW_FLASH_STORAGE_BASE - MICROPY_HW_ROMFS_BYTES) + static_assert(MICROPY_HW_FLASH_STORAGE_BYTES <= PICO_FLASH_SIZE_BYTES, "MICROPY_HW_FLASH_STORAGE_BYTES too big"); static_assert(MICROPY_HW_FLASH_STORAGE_BASE + MICROPY_HW_FLASH_STORAGE_BYTES <= PICO_FLASH_SIZE_BYTES, "MICROPY_HW_FLASH_STORAGE_BYTES too big"); @@ -53,6 +55,14 @@ typedef struct _rp2_flash_obj_t { uint32_t flash_size; } rp2_flash_obj_t; +#if MICROPY_HW_ROMFS_BYTES > 0 +static rp2_flash_obj_t rp2_flash_romfs_obj = { + .base = { &rp2_flash_type }, + .flash_base = MICROPY_HW_ROMFS_BASE, + .flash_size = MICROPY_HW_ROMFS_BYTES, +}; +#endif + static rp2_flash_obj_t rp2_flash_obj = { .base = { &rp2_flash_type }, .flash_base = MICROPY_HW_FLASH_STORAGE_BASE, @@ -138,6 +148,19 @@ static mp_obj_t rp2_flash_make_new(const mp_obj_type_t *type, size_t n_args, siz return MP_OBJ_FROM_PTR(self); } +static mp_int_t rp2_flash_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, mp_uint_t flags) { + rp2_flash_obj_t *self = MP_OBJ_TO_PTR(self_in); + if (flags == MP_BUFFER_READ) { + bufinfo->buf = (void *)(XIP_BASE + self->flash_base); + bufinfo->len = self->flash_size; + bufinfo->typecode = 'B'; + return 0; + } else { + // Write unsupported. + return 1; + } +} + static mp_obj_t rp2_flash_readblocks(size_t n_args, const mp_obj_t *args) { rp2_flash_obj_t *self = MP_OBJ_TO_PTR(args[0]); uint32_t offset = mp_obj_get_int(args[1]) * BLOCK_SIZE_BYTES; @@ -218,5 +241,21 @@ MP_DEFINE_CONST_OBJ_TYPE( MP_QSTR_Flash, MP_TYPE_FLAG_NONE, make_new, rp2_flash_make_new, + buffer, rp2_flash_get_buffer, locals_dict, &rp2_flash_locals_dict ); + +#if MICROPY_VFS_ROM_IOCTL +mp_obj_t mp_vfs_rom_ioctl(size_t n_args, const mp_obj_t *args) { + switch (mp_obj_get_int(args[0])) { + #if MICROPY_HW_ROMFS_BYTES > 0 + case MP_VFS_ROM_IOCTL_GET_NUMBER_OF_SEGMENTS: + return MP_OBJ_NEW_SMALL_INT(1); + case MP_VFS_ROM_IOCTL_GET_SEGMENT: + return MP_OBJ_FROM_PTR(&rp2_flash_romfs_obj); + #endif + default: + return MP_OBJ_NEW_SMALL_INT(-MP_EINVAL); + } +} +#endif diff --git a/ports/stm32/Makefile b/ports/stm32/Makefile index a9612f1a5792e..8ac9a8af03d1a 100644 --- a/ports/stm32/Makefile +++ b/ports/stm32/Makefile @@ -291,6 +291,7 @@ SRC_C += \ storage.c \ sdcard.c \ sdram.c \ + vfs_rom_ioctl.c \ fatfs_port.c \ lcd.c \ accel.c \ diff --git a/ports/stm32/boards/PYBD_SF2/f722_qspi.ld b/ports/stm32/boards/PYBD_SF2/f722_qspi.ld index 70d7f9cc4b8e4..05126233ec7ca 100644 --- a/ports/stm32/boards/PYBD_SF2/f722_qspi.ld +++ b/ports/stm32/boards/PYBD_SF2/f722_qspi.ld @@ -20,7 +20,8 @@ MEMORY { FLASH (rx) : ORIGIN = 0x08000000, LENGTH = 512K FLASH_APP (rx) : ORIGIN = 0x08008000, LENGTH = 480K /* sectors 2-7 */ - FLASH_EXT (rx) : ORIGIN = 0x90000000, LENGTH = 2048K /* external QSPI */ + FLASH_EXT (rx) : ORIGIN = 0x90000000, LENGTH = 1024K /* external QSPI */ + FLASH_ROMFS (rx): ORIGIN = 0x90100000, LENGTH = 1024K /* external QSPI */ RAM (rwx) : ORIGIN = 0x20000000, LENGTH = 256K /* DTCM+SRAM1+SRAM2 */ } @@ -39,6 +40,10 @@ _ram_end = ORIGIN(RAM) + LENGTH(RAM); _heap_start = _ebss; /* heap starts just after statically allocated memory */ _heap_end = _sstack; +/* ROMFS location */ +_micropy_hw_romfs_part1_start = ORIGIN(FLASH_ROMFS); +_micropy_hw_romfs_part1_size = LENGTH(FLASH_ROMFS); + /* Define output sections */ SECTIONS { diff --git a/ports/stm32/boards/PYBD_SF2/mpconfigboard.h b/ports/stm32/boards/PYBD_SF2/mpconfigboard.h index f0ef67e77e604..e8923bf9b66e5 100644 --- a/ports/stm32/boards/PYBD_SF2/mpconfigboard.h +++ b/ports/stm32/boards/PYBD_SF2/mpconfigboard.h @@ -64,6 +64,11 @@ void board_sleep(int value); #define MICROPY_HW_RTC_USE_US (1) #define MICROPY_HW_RTC_USE_CALOUT (1) +// ROMFS config +#define MICROPY_HW_ROMFS_ENABLE_EXTERNAL_QSPI (1) +#define MICROPY_HW_ROMFS_QSPI_SPIFLASH_OBJ (&spi_bdev2.spiflash) +#define MICROPY_HW_ROMFS_ENABLE_PART1 (1) + // SPI flash #1, for R/W storage #define MICROPY_HW_SOFTQSPI_SCK_LOW(self) (GPIOE->BSRR = (0x10000 << 11)) #define MICROPY_HW_SOFTQSPI_SCK_HIGH(self) (GPIOE->BSRR = (1 << 11)) diff --git a/ports/stm32/boards/PYBD_SF6/f767.ld b/ports/stm32/boards/PYBD_SF6/f767.ld index 4262a48a9962c..68da1f19d40dc 100644 --- a/ports/stm32/boards/PYBD_SF6/f767.ld +++ b/ports/stm32/boards/PYBD_SF6/f767.ld @@ -18,7 +18,7 @@ MEMORY { FLASH (rx) : ORIGIN = 0x08000000, LENGTH = 2048K FLASH_APP (rx) : ORIGIN = 0x08008000, LENGTH = 2016K /* sectors 1-11 3x32K 1*128K 7*256K */ - FLASH_EXT (rx) : ORIGIN = 0x90000000, LENGTH = 2048K /* external QSPI */ + FLASH_ROMFS (rx): ORIGIN = 0x90000000, LENGTH = 2048K /* external QSPI */ RAM (rwx) : ORIGIN = 0x20000000, LENGTH = 512K /* DTCM=128k, SRAM1=368K, SRAM2=16K */ } @@ -37,4 +37,8 @@ _ram_end = ORIGIN(RAM) + LENGTH(RAM); _heap_start = _ebss; /* heap starts just after statically allocated memory */ _heap_end = _sstack; +/* ROMFS location */ +_micropy_hw_romfs_part1_start = ORIGIN(FLASH_ROMFS); +_micropy_hw_romfs_part1_size = LENGTH(FLASH_ROMFS); + INCLUDE common_bl.ld diff --git a/ports/stm32/boards/PYBD_SF6/mpconfigboard.h b/ports/stm32/boards/PYBD_SF6/mpconfigboard.h index 8c11d7b2eec96..526a111177987 100644 --- a/ports/stm32/boards/PYBD_SF6/mpconfigboard.h +++ b/ports/stm32/boards/PYBD_SF6/mpconfigboard.h @@ -45,6 +45,11 @@ #define MICROPY_HW_CLK_PLLQ (6) #define MICROPY_HW_FLASH_LATENCY (FLASH_LATENCY_4) +// ROMFS config +#define MICROPY_HW_ROMFS_ENABLE_EXTERNAL_QSPI (1) +#define MICROPY_HW_ROMFS_QSPI_SPIFLASH_OBJ (&spi_bdev2.spiflash) +#define MICROPY_HW_ROMFS_ENABLE_PART1 (1) + // Extra UART config #define MICROPY_HW_UART7_TX (pyb_pin_W16) #define MICROPY_HW_UART7_RX (pyb_pin_W22B) diff --git a/ports/stm32/mpconfigboard_common.h b/ports/stm32/mpconfigboard_common.h index 4ce0a75b87e2b..2a650077f3a45 100644 --- a/ports/stm32/mpconfigboard_common.h +++ b/ports/stm32/mpconfigboard_common.h @@ -67,6 +67,26 @@ #define MICROPY_HW_ENTER_BOOTLOADER_VIA_RESET (1) #endif +// Whether to enable ROMFS on the internal flash. +#ifndef MICROPY_HW_ROMFS_ENABLE_INTERNAL_FLASH +#define MICROPY_HW_ROMFS_ENABLE_INTERNAL_FLASH (0) +#endif + +// Whether to enable ROMFS on external QSPI flash. +#ifndef MICROPY_HW_ROMFS_ENABLE_EXTERNAL_QSPI +#define MICROPY_HW_ROMFS_ENABLE_EXTERNAL_QSPI (0) +#endif + +// Whether to enable ROMFS partition 1. +#ifndef MICROPY_HW_ROMFS_ENABLE_PART1 +#define MICROPY_HW_ROMFS_ENABLE_PART1 (0) +#endif + +// Whether to enable ROMFS partition 2. +#ifndef MICROPY_HW_ROMFS_ENABLE_PART2 +#define MICROPY_HW_ROMFS_ENABLE_PART2 (0) +#endif + // Whether to enable storage on the internal flash of the MCU #ifndef MICROPY_HW_ENABLE_INTERNAL_FLASH_STORAGE #define MICROPY_HW_ENABLE_INTERNAL_FLASH_STORAGE (1) diff --git a/ports/stm32/mpconfigport.h b/ports/stm32/mpconfigport.h index 2b57446ee8233..da86fa641623a 100644 --- a/ports/stm32/mpconfigport.h +++ b/ports/stm32/mpconfigport.h @@ -79,6 +79,9 @@ #define MICROPY_SCHEDULER_STATIC_NODES (1) #define MICROPY_SCHEDULER_DEPTH (8) #define MICROPY_VFS (1) +#ifndef MICROPY_VFS_ROM +#define MICROPY_VFS_ROM (MICROPY_HW_ROMFS_ENABLE_INTERNAL_FLASH || MICROPY_HW_ROMFS_ENABLE_EXTERNAL_QSPI) +#endif // control over Python builtins #ifndef MICROPY_PY_BUILTINS_HELP_TEXT diff --git a/ports/stm32/qspi.c b/ports/stm32/qspi.c index 98364db41d4fb..34359a1ccf838 100644 --- a/ports/stm32/qspi.c +++ b/ports/stm32/qspi.c @@ -34,8 +34,6 @@ #if defined(MICROPY_HW_QSPIFLASH_SIZE_BITS_LOG2) -#define QSPI_MAP_ADDR (0x90000000) - #ifndef MICROPY_HW_QSPI_PRESCALER #define MICROPY_HW_QSPI_PRESCALER 3 // F_CLK = F_AHB/3 (72MHz when CPU is 216MHz) #endif diff --git a/ports/stm32/qspi.h b/ports/stm32/qspi.h index c774b12582be6..6fe91168cfdba 100644 --- a/ports/stm32/qspi.h +++ b/ports/stm32/qspi.h @@ -28,9 +28,16 @@ #include "drivers/bus/qspi.h" +#define QSPI_MAP_ADDR (0x90000000) +#define QSPI_MAP_ADDR_MAX (0xa0000000) + extern const mp_qspi_proto_t qspi_proto; void qspi_init(void); void qspi_memory_map(void); +static inline bool qspi_is_valid_addr(uint32_t addr) { + return QSPI_MAP_ADDR <= addr && addr < QSPI_MAP_ADDR_MAX; +} + #endif // MICROPY_INCLUDED_STM32_QSPI_H diff --git a/ports/stm32/vfs_rom_ioctl.c b/ports/stm32/vfs_rom_ioctl.c new file mode 100644 index 0000000000000..5024d44e5adc8 --- /dev/null +++ b/ports/stm32/vfs_rom_ioctl.c @@ -0,0 +1,161 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2025 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/obj.h" +#include "py/objarray.h" +#include "py/mperrno.h" +#include "extmod/vfs.h" +#include "drivers/memory/spiflash.h" + +#include "flash.h" +#include "qspi.h" +#include "storage.h" + +#if MICROPY_VFS_ROM_IOCTL + +#if MICROPY_HW_ROMFS_ENABLE_PART1 && !defined(MICROPY_HW_ROMFS_PART1_START) +#define MICROPY_HW_ROMFS_PART1_START (uintptr_t)(&_micropy_hw_romfs_part1_start) +#define MICROPY_HW_ROMFS_PART1_SIZE (uintptr_t)(&_micropy_hw_romfs_part1_size) +extern uint8_t _micropy_hw_romfs_part1_start; +extern uint8_t _micropy_hw_romfs_part1_size; +#endif + +#if MICROPY_HW_ROMFS_ENABLE_PART2 && !defined(MICROPY_HW_ROMFS_PART2_START) +#define MICROPY_HW_ROMFS_PART2_START (uintptr_t)(&_micropy_hw_romfs_part2_start) +#define MICROPY_HW_ROMFS_PART2_SIZE (uintptr_t)(&_micropy_hw_romfs_part2_size) +extern uint8_t _micropy_hw_romfs_part2_start; +extern uint8_t _micropy_hw_romfs_part2_size; +#endif + +#define ROMFS_MEMORYVIEW(base, size) {{&mp_type_memoryview}, 'B', 0, (size), (void *)(base)} + +static const mp_obj_array_t romfs_obj_table[] = { + #if MICROPY_HW_ROMFS_ENABLE_PART1 + ROMFS_MEMORYVIEW(MICROPY_HW_ROMFS_PART1_START, MICROPY_HW_ROMFS_PART1_SIZE), + #endif + #if MICROPY_HW_ROMFS_ENABLE_PART2 + ROMFS_MEMORYVIEW(MICROPY_HW_ROMFS_PART2_START, MICROPY_HW_ROMFS_PART2_SIZE), + #endif +}; + +mp_obj_t mp_vfs_rom_ioctl(size_t n_args, const mp_obj_t *args) { + mp_int_t cmd = mp_obj_get_int(args[0]); + if (cmd == MP_VFS_ROM_IOCTL_GET_NUMBER_OF_SEGMENTS) { + return MP_OBJ_NEW_SMALL_INT(MP_ARRAY_SIZE(romfs_obj_table)); + } + + if (n_args < 2) { + return MP_OBJ_NEW_SMALL_INT(-MP_EINVAL); + } + + mp_int_t romfs_id = mp_obj_get_int(args[1]); + if (!(0 <= romfs_id && romfs_id < MP_ARRAY_SIZE(romfs_obj_table))) { + return MP_OBJ_NEW_SMALL_INT(-MP_EINVAL); + } + + const mp_obj_array_t *romfs_obj = &romfs_obj_table[romfs_id]; + uintptr_t romfs_base = (uintptr_t)romfs_obj->items; + uintptr_t romfs_len = romfs_obj->len; + + if (cmd == MP_VFS_ROM_IOCTL_GET_SEGMENT) { + // Return the ROMFS memoryview object. + return MP_OBJ_FROM_PTR(romfs_obj); + } + + if (cmd == MP_VFS_ROM_IOCTL_WRITE_PREPARE) { + // Erase sectors in given range. + if (n_args < 3) { + return MP_OBJ_NEW_SMALL_INT(-MP_EINVAL); + } + uint32_t dest = romfs_base; + uint32_t dest_max = dest + mp_obj_get_int(args[2]); + if (dest_max > romfs_base + romfs_len) { + return MP_OBJ_NEW_SMALL_INT(-MP_EINVAL); + } + + #if MICROPY_HW_ROMFS_ENABLE_INTERNAL_FLASH + if (flash_is_valid_addr(dest)) { + while (dest < dest_max) { + int ret = flash_erase(dest); + if (ret < 0) { + return MP_OBJ_NEW_SMALL_INT(ret); + } + uint32_t sector_size = 0; + flash_get_sector_info(dest, NULL, §or_size); + dest += sector_size; + } + return MP_OBJ_NEW_SMALL_INT(16); + } + #endif + + #if MICROPY_HW_ROMFS_ENABLE_EXTERNAL_QSPI + if (qspi_is_valid_addr(dest)) { + dest -= QSPI_MAP_ADDR; + dest_max -= QSPI_MAP_ADDR; + while (dest < dest_max) { + int ret = mp_spiflash_erase_block(MICROPY_HW_ROMFS_QSPI_SPIFLASH_OBJ, dest); + if (ret < 0) { + return MP_OBJ_NEW_SMALL_INT(ret); + } + dest += MP_SPIFLASH_ERASE_BLOCK_SIZE; + } + return MP_OBJ_NEW_SMALL_INT(4); + } + #endif + } + + if (cmd == MP_VFS_ROM_IOCTL_WRITE) { + // Write data to flash. + if (n_args < 4) { + return MP_OBJ_NEW_SMALL_INT(-MP_EINVAL); + } + uint32_t dest = romfs_base + mp_obj_get_int(args[2]); + mp_buffer_info_t bufinfo; + mp_get_buffer_raise(args[3], &bufinfo, MP_BUFFER_READ); + if (dest + bufinfo.len > romfs_base + romfs_len) { + return MP_OBJ_NEW_SMALL_INT(-MP_EINVAL); + } + + #if MICROPY_HW_ROMFS_ENABLE_INTERNAL_FLASH + if (flash_is_valid_addr(dest)) { + int ret = flash_write(dest, bufinfo.buf, bufinfo.len / 4); + return MP_OBJ_NEW_SMALL_INT(ret); + } + #endif + + #if MICROPY_HW_ROMFS_ENABLE_EXTERNAL_QSPI + if (qspi_is_valid_addr(dest)) { + dest -= QSPI_MAP_ADDR; + int ret = mp_spiflash_write(MICROPY_HW_ROMFS_QSPI_SPIFLASH_OBJ, dest, bufinfo.len, bufinfo.buf); + return MP_OBJ_NEW_SMALL_INT(ret); + } + #endif + } + + return MP_OBJ_NEW_SMALL_INT(-MP_EINVAL); +} + +#endif // MICROPY_VFS_ROM_IOCTL diff --git a/ports/unix/main.c b/ports/unix/main.c index 58fa3ff589a55..0d137fe1b58ea 100644 --- a/ports/unix/main.c +++ b/ports/unix/main.c @@ -45,6 +45,7 @@ #include "py/gc.h" #include "py/objstr.h" #include "py/cstack.h" +#include "py/mperrno.h" #include "py/mphal.h" #include "py/mpthread.h" #include "extmod/misc.h" @@ -548,7 +549,14 @@ MP_NOINLINE int main_(int argc, char **argv) { MP_OBJ_NEW_QSTR(MP_QSTR__slash_), }; mp_vfs_mount(2, args, (mp_map_t *)&mp_const_empty_map); + + // Make sure the root that was just mounted is the current VFS (it's always at + // the end of the linked list). Can't use chdir('/') because that will change + // the current path within the VfsPosix object. MP_STATE_VM(vfs_cur) = MP_STATE_VM(vfs_mount_table); + while (MP_STATE_VM(vfs_cur)->next != NULL) { + MP_STATE_VM(vfs_cur) = MP_STATE_VM(vfs_cur)->next; + } } #endif @@ -803,3 +811,22 @@ void nlr_jump_fail(void *val) { fprintf(stderr, "FATAL: uncaught NLR %p\n", val); exit(1); } + +#if MICROPY_VFS_ROM_IOCTL + +static uint8_t romfs_buf[4] = { 0xd2, 0xcd, 0x31, 0x00 }; // empty ROMFS +static const MP_DEFINE_MEMORYVIEW_OBJ(romfs_obj, 'B', 0, sizeof(romfs_buf), romfs_buf); + +mp_obj_t mp_vfs_rom_ioctl(size_t n_args, const mp_obj_t *args) { + switch (mp_obj_get_int(args[0])) { + case MP_VFS_ROM_IOCTL_GET_NUMBER_OF_SEGMENTS: + return MP_OBJ_NEW_SMALL_INT(1); + + case MP_VFS_ROM_IOCTL_GET_SEGMENT: + return MP_OBJ_FROM_PTR(&romfs_obj); + } + + return MP_OBJ_NEW_SMALL_INT(-MP_EINVAL); +} + +#endif diff --git a/ports/unix/variants/coverage/mpconfigvariant.h b/ports/unix/variants/coverage/mpconfigvariant.h index e24e0f1b14ade..a72e533e74c4a 100644 --- a/ports/unix/variants/coverage/mpconfigvariant.h +++ b/ports/unix/variants/coverage/mpconfigvariant.h @@ -41,4 +41,6 @@ #define MICROPY_DEBUG_PARSE_RULE_NAME (1) #define MICROPY_TRACKED_ALLOC (1) #define MICROPY_WARNINGS_CATEGORY (1) +#undef MICROPY_VFS_ROM_IOCTL +#define MICROPY_VFS_ROM_IOCTL (1) #define MICROPY_PY_CRYPTOLIB_CTR (1) diff --git a/ports/unix/variants/mpconfigvariant_common.h b/ports/unix/variants/mpconfigvariant_common.h index 093477af0aded..9eeed8797366c 100644 --- a/ports/unix/variants/mpconfigvariant_common.h +++ b/ports/unix/variants/mpconfigvariant_common.h @@ -123,3 +123,4 @@ #define MICROPY_PY_MACHINE_PIN_BASE (1) #define MICROPY_VFS_ROM (1) +#define MICROPY_VFS_ROM_IOCTL (0) diff --git a/py/mpconfig.h b/py/mpconfig.h index 66b3d125e78de..05f39b3605bf3 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -1006,6 +1006,11 @@ typedef double mp_float_t; #define MICROPY_VFS_WRITABLE (1) #endif +// Whether to enable the mp_vfs_rom_ioctl C function, and vfs.rom_ioctl Python function +#ifndef MICROPY_VFS_ROM_IOCTL +#define MICROPY_VFS_ROM_IOCTL (MICROPY_VFS_ROM) +#endif + // Support for VFS POSIX component, to mount a POSIX filesystem within VFS #ifndef MICROPY_VFS_POSIX #define MICROPY_VFS_POSIX (0) diff --git a/py/objarray.h b/py/objarray.h index 4a0e8a983fe77..bb7a514b97913 100644 --- a/py/objarray.h +++ b/py/objarray.h @@ -53,6 +53,10 @@ typedef struct _mp_obj_array_t { } mp_obj_array_t; #if MICROPY_PY_BUILTINS_MEMORYVIEW + +#define MP_DEFINE_MEMORYVIEW_OBJ(obj_name, typecode, offset, len, ptr) \ + mp_obj_array_t obj_name = {{&mp_type_memoryview}, (typecode), (offset), (len), (ptr)} + static inline void mp_obj_memoryview_init(mp_obj_array_t *self, size_t typecode, size_t offset, size_t len, void *items) { self->base.type = &mp_type_memoryview; self->typecode = typecode; @@ -60,6 +64,7 @@ static inline void mp_obj_memoryview_init(mp_obj_array_t *self, size_t typecode, self->len = len; self->items = items; } + #endif #if MICROPY_PY_ARRAY || MICROPY_PY_BUILTINS_BYTEARRAY diff --git a/py/qstrdefs.h b/py/qstrdefs.h index 5003636df3d4b..0b50d279f9dbf 100644 --- a/py/qstrdefs.h +++ b/py/qstrdefs.h @@ -68,6 +68,11 @@ Q(utf-8) Q(.frozen) #endif +#if MICROPY_VFS_ROM && MICROPY_VFS_ROM_IOCTL +Q(/rom) +Q(/rom/lib) +#endif + #if MICROPY_ENABLE_PYSTACK Q(pystack exhausted) #endif diff --git a/py/runtime.c b/py/runtime.c index bf26921f9ab9f..58819819ad025 100644 --- a/py/runtime.c +++ b/py/runtime.c @@ -46,6 +46,10 @@ #include "py/cstack.h" #include "py/gc.h" +#if MICROPY_VFS_ROM && MICROPY_VFS_ROM_IOCTL +#include "extmod/vfs.h" +#endif + #if MICROPY_DEBUG_VERBOSE // print debugging info #define DEBUG_PRINT (1) #define DEBUG_printf DEBUG_printf @@ -185,6 +189,11 @@ void mp_init(void) { #endif MP_THREAD_GIL_ENTER(); + + #if MICROPY_VFS_ROM && MICROPY_VFS_ROM_IOCTL + // Mount ROMFS if it exists. + mp_vfs_mount_romfs_protected(); + #endif } void mp_deinit(void) { diff --git a/tools/mpremote/mpremote/commands.py b/tools/mpremote/mpremote/commands.py index f86befd080349..7dd448c8b771c 100644 --- a/tools/mpremote/mpremote/commands.py +++ b/tools/mpremote/mpremote/commands.py @@ -1,12 +1,15 @@ +import binascii import hashlib import os import sys import tempfile +import zlib import serial.tools.list_ports -from .transport import TransportError, stdout_write_bytes +from .transport import TransportError, TransportExecError, stdout_write_bytes from .transport_serial import SerialTransport +from .romfs import make_romfs class CommandError(Exception): @@ -478,3 +481,181 @@ def do_rtc(state, args): state.transport.exec("machine.RTC().datetime({})".format(timetuple)) else: print(state.transport.eval("machine.RTC().datetime()")) + + +def _do_romfs_query(state, args): + state.ensure_raw_repl() + state.did_action() + + # Detect the romfs and get its associated device. + state.transport.exec("import vfs") + if not state.transport.eval("hasattr(vfs,'rom_ioctl')"): + print("ROMFS is not enabled on this device") + return + num_rom_partitions = state.transport.eval("vfs.rom_ioctl(1)") + if num_rom_partitions <= 0: + print("No ROMFS partitions available") + return + + for rom_id in range(num_rom_partitions): + state.transport.exec(f"dev=vfs.rom_ioctl(2,{rom_id})") + has_object = state.transport.eval("hasattr(dev,'ioctl')") + if has_object: + rom_block_count = state.transport.eval("dev.ioctl(4,0)") + rom_block_size = state.transport.eval("dev.ioctl(5,0)") + rom_size = rom_block_count * rom_block_size + print( + f"ROMFS{rom_id} partition has size {rom_size} bytes ({rom_block_count} blocks of {rom_block_size} bytes each)" + ) + else: + rom_size = state.transport.eval("len(dev)") + print(f"ROMFS{rom_id} partition has size {rom_size} bytes") + romfs = state.transport.eval("bytes(memoryview(dev)[:12])") + print(f" Raw contents: {romfs.hex(':')} ...") + if not romfs.startswith(b"\xd2\xcd\x31"): + print(" Not a valid ROMFS") + else: + size = 0 + for value in romfs[3:]: + size = (size << 7) | (value & 0x7F) + if not value & 0x80: + break + print(f" ROMFS image size: {size}") + + +def _do_romfs_build(state, args): + state.did_action() + + if args.path is None: + raise CommandError("romfs build: source path not given") + + input_directory = args.path + + if args.output is None: + output_file = input_directory + ".romfs" + else: + output_file = args.output + + romfs = make_romfs(input_directory, mpy_cross=args.mpy) + + print(f"Writing {len(romfs)} bytes to output file {output_file}") + with open(output_file, "wb") as f: + f.write(romfs) + + +def _do_romfs_deploy(state, args): + state.ensure_raw_repl() + state.did_action() + transport = state.transport + + if args.path is None: + raise CommandError("romfs deploy: source path not given") + + rom_id = args.partition + romfs_filename = args.path + + # Read in or create the ROMFS filesystem image. + if romfs_filename.endswith(".romfs"): + with open(romfs_filename, "rb") as f: + romfs = f.read() + else: + romfs = make_romfs(romfs_filename, mpy_cross=args.mpy) + print(f"Image size is {len(romfs)} bytes") + + # Detect the ROMFS partition and get its associated device. + state.transport.exec("import vfs") + if not state.transport.eval("hasattr(vfs,'rom_ioctl')"): + raise CommandError("ROMFS is not enabled on this device") + transport.exec(f"dev=vfs.rom_ioctl(2,{rom_id})") + if transport.eval("isinstance(dev,int) and dev<0"): + raise CommandError(f"ROMFS{rom_id} partition not found on device") + has_object = transport.eval("hasattr(dev,'ioctl')") + if has_object: + rom_block_count = transport.eval("dev.ioctl(4,0)") + rom_block_size = transport.eval("dev.ioctl(5,0)") + rom_size = rom_block_count * rom_block_size + print( + f"ROMFS{rom_id} partition has size {rom_size} bytes ({rom_block_count} blocks of {rom_block_size} bytes each)" + ) + else: + rom_size = transport.eval("len(dev)") + print(f"ROMFS{rom_id} partition has size {rom_size} bytes") + + # Check if ROMFS filesystem image will fit in the target partition. + if len(romfs) > rom_size: + print("ROMFS image is too big for the target partition") + sys.exit(1) + + # Prepare ROMFS partition for writing. + print(f"Preparing ROMFS{rom_id} partition for writing") + transport.exec("import vfs\ntry:\n vfs.umount('/rom')\nexcept:\n pass") + chunk_size = 4096 + if has_object: + for offset in range(0, len(romfs), rom_block_size): + transport.exec(f"dev.ioctl(6,{offset // rom_block_size})") + chunk_size = min(chunk_size, rom_block_size) + else: + rom_min_write = transport.eval(f"vfs.rom_ioctl(3,{rom_id},{len(romfs)})") + chunk_size = max(chunk_size, rom_min_write) + + # Detect capabilities of the device to use the fastest method of transfer. + has_bytes_fromhex = transport.eval("hasattr(bytes,'fromhex')") + try: + transport.exec("from binascii import a2b_base64") + has_a2b_base64 = True + except TransportExecError: + has_a2b_base64 = False + try: + transport.exec("from io import BytesIO") + transport.exec("from deflate import DeflateIO,RAW") + has_deflate_io = True + except TransportExecError: + has_deflate_io = False + + # Deploy the ROMFS filesystem image to the device. + for offset in range(0, len(romfs), chunk_size): + romfs_chunk = romfs[offset : offset + chunk_size] + romfs_chunk += bytes(chunk_size - len(romfs_chunk)) + if has_deflate_io: + # Needs: binascii.a2b_base64, io.BytesIO, deflate.DeflateIO. + romfs_chunk_compressed = zlib.compress(romfs_chunk, wbits=-9) + buf = binascii.b2a_base64(romfs_chunk_compressed).strip() + transport.exec(f"buf=DeflateIO(BytesIO(a2b_base64({buf})),RAW,9).read()") + elif has_a2b_base64: + # Needs: binascii.a2b_base64. + buf = binascii.b2a_base64(romfs_chunk) + transport.exec(f"buf=a2b_base64({buf})") + elif has_bytes_fromhex: + # Needs: bytes.fromhex. + buf = romfs_chunk.hex() + transport.exec(f"buf=bytes.fromhex('{buf}')") + else: + # Needs nothing special. + transport.exec("buf=" + repr(romfs_chunk)) + print(f"\rWriting at offset {offset}", end="") + if has_object: + transport.exec( + f"dev.writeblocks({offset // rom_block_size},buf,{offset % rom_block_size})" + ) + else: + transport.exec(f"vfs.rom_ioctl(4,{rom_id},{offset},buf)") + + # Complete writing. + if not has_object: + transport.eval(f"vfs.rom_ioctl(5,{rom_id})") + + print() + print("ROMFS image deployed") + + +def do_romfs(state, args): + if args.command[0] == "query": + _do_romfs_query(state, args) + elif args.command[0] == "build": + _do_romfs_build(state, args) + elif args.command[0] == "deploy": + _do_romfs_deploy(state, args) + else: + raise CommandError( + f"romfs: '{args.command[0]}' is not a command; pass romfs --help for a list" + ) diff --git a/tools/mpremote/mpremote/main.py b/tools/mpremote/mpremote/main.py index e6e397020fe85..bdae8f163656b 100644 --- a/tools/mpremote/mpremote/main.py +++ b/tools/mpremote/mpremote/main.py @@ -36,6 +36,7 @@ do_resume, do_rtc, do_soft_reset, + do_romfs, ) from .mip import do_mip from .repl import do_repl @@ -228,6 +229,32 @@ def argparse_mip(): return cmd_parser +def argparse_romfs(): + cmd_parser = argparse.ArgumentParser(description="manage ROM partitions") + _bool_flag( + cmd_parser, + "mpy", + "m", + True, + "automatically compile .py files to .mpy when building the ROMFS image (default)", + ) + cmd_parser.add_argument( + "--partition", + "-p", + type=int, + default=0, + help="ROMFS partition to use", + ) + cmd_parser.add_argument( + "--output", + "-o", + help="output file", + ) + cmd_parser.add_argument("command", nargs=1, help="romfs command, one of: query, build, deploy") + cmd_parser.add_argument("path", nargs="?", help="path to directory to deploy") + return cmd_parser + + def argparse_none(description): return lambda: argparse.ArgumentParser(description=description) @@ -302,6 +329,10 @@ def argparse_none(description): do_version, argparse_none("print version and exit"), ), + "romfs": ( + do_romfs, + argparse_romfs, + ), } # Additional commands aliases. diff --git a/tools/mpremote/mpremote/romfs.py b/tools/mpremote/mpremote/romfs.py new file mode 100644 index 0000000000000..ae781a36dfe62 --- /dev/null +++ b/tools/mpremote/mpremote/romfs.py @@ -0,0 +1,148 @@ +# MIT license; Copyright (c) 2022 Damien P. George + +import struct, sys, os + +try: + from mpy_cross import run as mpy_cross_run +except ImportError: + mpy_cross_run = None + + +class VfsRomWriter: + ROMFS_HEADER = b"\xd2\xcd\x31" + + ROMFS_RECORD_KIND_UNUSED = 0 + ROMFS_RECORD_KIND_PADDING = 1 + ROMFS_RECORD_KIND_DATA_VERBATIM = 2 + ROMFS_RECORD_KIND_DATA_POINTER = 3 + ROMFS_RECORD_KIND_DIRECTORY = 4 + ROMFS_RECORD_KIND_FILE = 5 + + def __init__(self): + self._dir_stack = [(None, bytearray())] + + def _encode_uint(self, value): + encoded = [value & 0x7F] + value >>= 7 + while value != 0: + encoded.insert(0, 0x80 | (value & 0x7F)) + value >>= 7 + return bytes(encoded) + + def _pack(self, kind, payload): + return self._encode_uint(kind) + self._encode_uint(len(payload)) + payload + + def _extend(self, data): + buf = self._dir_stack[-1][1] + buf.extend(data) + return len(buf) + + def finalise(self): + _, data = self._dir_stack.pop() + encoded_kind = VfsRomWriter.ROMFS_HEADER + encoded_len = self._encode_uint(len(data)) + if (len(encoded_kind) + len(encoded_len) + len(data)) % 2 == 1: + encoded_len = b"\x80" + encoded_len + data = encoded_kind + encoded_len + data + return data + + def opendir(self, dirname): + self._dir_stack.append((dirname, bytearray())) + + def closedir(self): + dirname, dirdata = self._dir_stack.pop() + dirdata = self._encode_uint(len(dirname)) + bytes(dirname, "ascii") + dirdata + self._extend(self._pack(VfsRomWriter.ROMFS_RECORD_KIND_DIRECTORY, dirdata)) + + def mkdata(self, data): + assert len(self._dir_stack) == 1 + return self._extend(self._pack(VfsRomWriter.ROMFS_RECORD_KIND_DATA_VERBATIM, data)) - len( + data + ) + + def mkfile(self, filename, filedata): + filename = bytes(filename, "ascii") + payload = self._encode_uint(len(filename)) + payload += filename + if isinstance(filedata, tuple): + sub_payload = self._encode_uint(filedata[0]) + sub_payload += self._encode_uint(filedata[1]) + payload += self._pack(VfsRomWriter.ROMFS_RECORD_KIND_DATA_POINTER, sub_payload) + else: + payload += self._pack(VfsRomWriter.ROMFS_RECORD_KIND_DATA_VERBATIM, filedata) + self._dir_stack[-1][1].extend(self._pack(VfsRomWriter.ROMFS_RECORD_KIND_FILE, payload)) + + +def copy_recursively(vfs, src_dir, print_prefix, mpy_cross): + assert src_dir.endswith("/") + DIR = 1 << 14 + mpy_cross_missed = 0 + dir_contents = sorted(os.listdir(src_dir)) + for name in dir_contents: + src_name = src_dir + name + st = os.stat(src_name) + + if name == dir_contents[-1]: + # Last entry in the directory listing. + print_entry = "\\--" + print_recurse = " " + else: + # Not the last entry in the directory listing. + print_entry = "|--" + print_recurse = "| " + + if st[0] & DIR: + # A directory, enter it and copy its contents recursively. + print(print_prefix + print_entry, name + "/") + vfs.opendir(name) + mpy_cross_missed += copy_recursively( + vfs, src_name + "/", print_prefix + print_recurse, mpy_cross + ) + vfs.closedir() + else: + # A file. + did_mpy = False + name_extra = "" + if mpy_cross and name.endswith(".py"): + name_mpy = name[:-3] + ".mpy" + src_name_mpy = src_dir + name_mpy + if not os.path.isfile(src_name_mpy): + if mpy_cross_run is not None: + did_mpy = True + proc = mpy_cross_run(src_name) + proc.wait() + else: + mpy_cross_missed += 1 + if did_mpy: + name_extra = " -> .mpy" + print(print_prefix + print_entry, name + name_extra) + if did_mpy: + name = name_mpy + src_name = src_name_mpy + with open(src_name, "rb") as src: + vfs.mkfile(name, src.read()) + if did_mpy: + os.remove(src_name_mpy) + return mpy_cross_missed + + +def make_romfs(src_dir, *, mpy_cross): + if not src_dir.endswith("/"): + src_dir += "/" + + vfs = VfsRomWriter() + + # Build the filesystem recursively. + print("Building romfs filesystem, source directory: {}".format(src_dir)) + print("/") + try: + mpy_cross_missed = copy_recursively(vfs, src_dir, "", mpy_cross) + except OSError as er: + print("Error: OSError {}".format(er), file=sys.stderr) + sys.exit(1) + + if mpy_cross_missed: + print("Warning: `mpy_cross` module not found, .py files were not precompiled") + mpy_cross = False + + return vfs.finalise()