diff --git a/extmod/extmod.cmake b/extmod/extmod.cmake index 98e8a84608a56..154931a664477 100644 --- a/extmod/extmod.cmake +++ b/extmod/extmod.cmake @@ -58,6 +58,8 @@ set(MICROPY_SOURCE_EXTMOD ${MICROPY_EXTMOD_DIR}/vfs_fat_diskio.c ${MICROPY_EXTMOD_DIR}/vfs_fat_file.c ${MICROPY_EXTMOD_DIR}/vfs_lfs.c + ${MICROPY_EXTMOD_DIR}/vfs_rom.c + ${MICROPY_EXTMOD_DIR}/vfs_rom_file.c ${MICROPY_EXTMOD_DIR}/vfs_posix.c ${MICROPY_EXTMOD_DIR}/vfs_posix_file.c ${MICROPY_EXTMOD_DIR}/vfs_reader.c diff --git a/extmod/extmod.mk b/extmod/extmod.mk index f2ae3ad6923fb..c8bc74d268e77 100644 --- a/extmod/extmod.mk +++ b/extmod/extmod.mk @@ -61,6 +61,8 @@ SRC_EXTMOD_C += \ extmod/vfs_fat_diskio.c \ extmod/vfs_fat_file.c \ extmod/vfs_lfs.c \ + extmod/vfs_rom.c \ + extmod/vfs_rom_file.c \ extmod/vfs_posix.c \ extmod/vfs_posix_file.c \ extmod/vfs_reader.c \ diff --git a/extmod/modvfs.c b/extmod/modvfs.c index 32dc0e5d08b0d..df422365b75ef 100644 --- a/extmod/modvfs.c +++ b/extmod/modvfs.c @@ -32,6 +32,7 @@ #include "extmod/vfs_fat.h" #include "extmod/vfs_lfs.h" #include "extmod/vfs_posix.h" +#include "extmod/vfs_rom.h" #if !MICROPY_VFS #error "MICROPY_PY_VFS requires MICROPY_VFS" @@ -51,6 +52,9 @@ static const mp_rom_map_elem_t vfs_module_globals_table[] = { #if MICROPY_VFS_LFS2 { MP_ROM_QSTR(MP_QSTR_VfsLfs2), MP_ROM_PTR(&mp_type_vfs_lfs2) }, #endif + #if MICROPY_VFS_ROM + { MP_ROM_QSTR(MP_QSTR_VfsRom), MP_ROM_PTR(&mp_type_vfs_rom) }, + #endif #if MICROPY_VFS_POSIX { MP_ROM_QSTR(MP_QSTR_VfsPosix), MP_ROM_PTR(&mp_type_vfs_posix) }, #endif diff --git a/extmod/vfs_reader.c b/extmod/vfs_reader.c index 80d0fa6344aa1..de5c4e03d3c13 100644 --- a/extmod/vfs_reader.c +++ b/extmod/vfs_reader.c @@ -85,6 +85,17 @@ void mp_reader_new_file(mp_reader_t *reader, qstr filename) { const mp_stream_p_t *stream_p = mp_get_stream(file); int errcode = 0; + + #if MICROPY_VFS_ROM + // Check if the stream can be memory mapped. + mp_buffer_info_t bufinfo; + if (mp_get_buffer(file, &bufinfo, MP_BUFFER_READ)) { + mp_reader_new_mem(reader, bufinfo.buf, bufinfo.len, MP_READER_IS_ROM); + return; + } + #endif + + // Determine how big the input buffer should be, if the stream requests a certain size or not. mp_uint_t bufsize = stream_p->ioctl(file, MP_STREAM_GET_BUFFER_SIZE, 0, &errcode); if (bufsize == MP_STREAM_ERROR || bufsize == 0) { // bufsize == 0 is included here to support mpremote v1.21 and older where mount file ioctl @@ -94,6 +105,7 @@ void mp_reader_new_file(mp_reader_t *reader, qstr filename) { bufsize = MIN(MICROPY_READER_VFS_MAX_BUFFER_SIZE, MAX(MICROPY_READER_VFS_MIN_BUFFER_SIZE, bufsize)); } + // Create the reader. mp_reader_vfs_t *rf = m_new_obj_var(mp_reader_vfs_t, buf, byte, bufsize); rf->file = file; rf->bufsize = bufsize; diff --git a/extmod/vfs_rom.c b/extmod/vfs_rom.c new file mode 100644 index 0000000000000..b8b6f8e40d12b --- /dev/null +++ b/extmod/vfs_rom.c @@ -0,0 +1,399 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2022 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +// ROMFS filesystem format +// ======================= +// +// ROMFS is a flexible and extensible filesystem format designed to represent a +// directory hierarchy with files, where those files are read-only and their data +// can be memory mapped. +// +// Concepts: +// - varuint: An unsigned integer that is encoded in a variable number of bytes. It is +// stored big-endian with the high bit of the byte set if there are following bytes. +// - record: A variable sized element with a type. It is stored as two varuint's and then +// a payload. The first varuint is the record kind and the second varuint is the +// payload length (which may be zero bytes long). +// +// A ROMFS filesystem is a record with record kind 0x14a6b1, chosen so the encoded value +// is 0xd2-0xcd-0x31 which is "RM1" with the first two bytes having their high bit set. +// If the ROMFS record's payload is non-empty then it contains records. +// +// Record types: +// - 0 = unused, can be used to detect corruption of the filesystem. +// - 1 = padding/comments, can contain any data in their payload. +// - 2 = verbatim data, used to store file data. +// - 3 = indirect data, pointer to offset within the ROMFS payload. +// - 4 = a directory: payload contains a varuint which is the length of the directory +// name in bytes, then the name, then optional nested records for the contents +// of the directory (including optional metadata). +// - 5 = a file: payload contains a varuint which is the length of the filename in bytes +// then the name, then optional nested records. +// +// Remarks: +// - A varuint can be padded if needed by prepending with one or more 0x80 bytes. This +// padding does not change any semantics. +// - The size of the ROMFS record (including kind and length and payload) must be a +// multiple of 2 (because it's not possible to add a padding record of one byte). +// - File data can be optionally aligned using padding records and/or indirect data +// records. +// - There is no limit to the size of directory/file names or file data. +// +// Unknown record types must be skipped over. They may in the future add optional +// features, while still retaining backwards compatibility. Such features may be: +// - Alignment requirements of the ROMFS record. +// - Timestamps on directories/files. +// - A precomputed hash of a file, or other metadata. +// - An optimised lookup table indexing the directory hierarchy. + +#include + +#include "py/bc.h" +#include "py/runtime.h" +#include "py/mperrno.h" +#include "extmod/vfs.h" +#include "extmod/vfs_rom.h" + +#if MICROPY_VFS_ROM + +#define ROMFS_SIZE_MIN (4) +#define ROMFS_HEADER_BYTE0 (0x80 | 'R') +#define ROMFS_HEADER_BYTE1 (0x80 | 'M') +#define ROMFS_HEADER_BYTE2 (0x00 | '1') + +// Values for `record_kind_t`. +#define ROMFS_RECORD_KIND_UNUSED (0) +#define ROMFS_RECORD_KIND_PADDING (1) +#define ROMFS_RECORD_KIND_DATA_VERBATIM (2) +#define ROMFS_RECORD_KIND_DATA_POINTER (3) +#define ROMFS_RECORD_KIND_DIRECTORY (4) +#define ROMFS_RECORD_KIND_FILE (5) + +typedef mp_uint_t record_kind_t; + +struct _mp_obj_vfs_rom_t { + mp_obj_base_t base; + mp_obj_t memory; + const uint8_t *filesystem; + const uint8_t *filesystem_end; +}; + +static record_kind_t extract_record(const uint8_t **fs, const uint8_t **fs_next) { + record_kind_t record_kind = mp_decode_uint(fs); + mp_uint_t record_len = mp_decode_uint(fs); + *fs_next = *fs + record_len; + return record_kind; +} + +static void extract_data(mp_obj_vfs_rom_t *self, const uint8_t *fs, const uint8_t *fs_top, size_t *size_out, const uint8_t **data_out) { + *size_out = 0; + *data_out = NULL; + while (fs < fs_top) { + const uint8_t *fs_next; + record_kind_t record_kind = extract_record(&fs, &fs_next); + if (record_kind == ROMFS_RECORD_KIND_DATA_VERBATIM) { + // Verbatim data. + *size_out = fs_next - fs; + *data_out = fs; + break; + } else if (record_kind == ROMFS_RECORD_KIND_DATA_POINTER) { + // Pointer to data. + *size_out = mp_decode_uint(&fs); + *data_out = self->filesystem + mp_decode_uint(&fs); + break; + } else { + // Skip this record. + fs = fs_next; + } + } +} + +// Searches for `path` in the filesystem. +// `path` must be null-terminated. +mp_import_stat_t mp_vfs_rom_search_filesystem(mp_obj_vfs_rom_t *self, const char *path, size_t *size_out, const uint8_t **data_out) { + const uint8_t *fs = self->filesystem; + const uint8_t *fs_top = self->filesystem_end; + size_t path_len = strlen(path); + if (*path == '/') { + // An optional slash at the start of the path enters the top-level filesystem. + ++path; + --path_len; + } + while (path_len > 0 && fs < fs_top) { + const uint8_t *fs_next; + record_kind_t record_kind = extract_record(&fs, &fs_next); + if (record_kind == ROMFS_RECORD_KIND_DIRECTORY || record_kind == ROMFS_RECORD_KIND_FILE) { + // A directory or file record. + mp_uint_t name_len = mp_decode_uint(&fs); + if ((name_len == path_len + || (name_len < path_len && path[name_len] == '/')) + && memcmp(path, fs, name_len) == 0) { + // Name matches, so enter this record. + fs += name_len; + fs_top = fs_next; + path += name_len; + path_len -= name_len; + if (record_kind == ROMFS_RECORD_KIND_DIRECTORY) { + // Continue searching in this directory. + if (*path == '/') { + ++path; + --path_len; + } + } else { + // Return this file. + if (path_len != 0) { + return MP_IMPORT_STAT_NO_EXIST; + } + if (size_out != NULL) { + extract_data(self, fs, fs_top, size_out, data_out); + } + return MP_IMPORT_STAT_FILE; + } + } else { + // Skip this directory/file record. + fs = fs_next; + } + } else { + // Skip this record. + fs = fs_next; + } + } + if (path_len == 0) { + if (size_out != NULL) { + *size_out = fs_top - fs; + *data_out = fs; + } + return MP_IMPORT_STAT_DIR; + } + return MP_IMPORT_STAT_NO_EXIST; +} + +static mp_obj_t vfs_rom_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { + mp_arg_check_num(n_args, n_kw, 1, 1, false); + + mp_obj_vfs_rom_t *self = m_new_obj(mp_obj_vfs_rom_t); + self->base.type = type; + self->memory = args[0]; + + mp_buffer_info_t bufinfo; + if (mp_get_buffer(self->memory, &bufinfo, MP_BUFFER_READ)) { + if (bufinfo.len < ROMFS_SIZE_MIN) { + mp_raise_OSError(MP_ENODEV); + } + self->filesystem = bufinfo.buf; + } else { + self->filesystem = (const uint8_t *)(uintptr_t)mp_obj_get_int_truncated(self->memory); + } + + // Verify it is a ROMFS. + if (!(self->filesystem[0] == ROMFS_HEADER_BYTE0 + && self->filesystem[1] == ROMFS_HEADER_BYTE1 + && self->filesystem[2] == ROMFS_HEADER_BYTE2)) { + mp_raise_OSError(MP_ENODEV); + } + + // The ROMFS is a record itself, so enter into it and compute its limit. + extract_record(&self->filesystem, &self->filesystem_end); + + return MP_OBJ_FROM_PTR(self); +} + +static mp_obj_t vfs_rom_mount(mp_obj_t self_in, mp_obj_t readonly, mp_obj_t mkfs) { + (void)self_in; + (void)readonly; + if (mp_obj_is_true(mkfs)) { + mp_raise_OSError(MP_EPERM); + } + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_3(vfs_rom_mount_obj, vfs_rom_mount); + +// mp_vfs_rom_file_open is implemented in vfs_rom_file.c. +static MP_DEFINE_CONST_FUN_OBJ_3(vfs_rom_open_obj, mp_vfs_rom_file_open); + +static mp_obj_t vfs_rom_chdir(mp_obj_t self_in, mp_obj_t path_in) { + mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in); + const char *path = mp_vfs_rom_get_path_str(self, path_in); + if (path[0] == '/' && path[1] == '\0') { + // Allow chdir to the root of the filesystem. + } else { + // Don't allow chdir to any subdirectory (not currently implemented). + mp_raise_OSError(MP_EOPNOTSUPP); + } + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_2(vfs_rom_chdir_obj, vfs_rom_chdir); + +typedef struct _vfs_rom_ilistdir_it_t { + mp_obj_base_t base; + mp_fun_1_t iternext; + mp_obj_vfs_rom_t *vfs_rom; + bool is_str; + const uint8_t *index; + const uint8_t *index_top; +} vfs_rom_ilistdir_it_t; + +static mp_obj_t vfs_rom_ilistdir_it_iternext(mp_obj_t self_in) { + vfs_rom_ilistdir_it_t *self = MP_OBJ_TO_PTR(self_in); + + while (self->index < self->index_top) { + const uint8_t *index_next; + record_kind_t record_kind = extract_record(&self->index, &index_next); + uint32_t type; + mp_uint_t name_len; + size_t data_len; + if (record_kind == ROMFS_RECORD_KIND_DIRECTORY || record_kind == ROMFS_RECORD_KIND_FILE) { + // A directory or file record. + name_len = mp_decode_uint(&self->index); + if (record_kind == ROMFS_RECORD_KIND_DIRECTORY) { + // A directory. + type = MP_S_IFDIR; + data_len = index_next - self->index - name_len; + } else { + // A file. + type = MP_S_IFREG; + const uint8_t *data_value; + extract_data(self->vfs_rom, self->index + name_len, index_next, &data_len, &data_value); + } + } else { + // Skip this record. + self->index = index_next; + continue; + } + + const uint8_t *name_str = self->index; + self->index = index_next; + + // Make 4-tuple with info about this entry: (name, attr, inode, size) + mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(4, NULL)); + + if (self->is_str) { + t->items[0] = mp_obj_new_str((const char *)name_str, name_len); + } else { + t->items[0] = mp_obj_new_bytes(name_str, name_len); + } + + t->items[1] = MP_OBJ_NEW_SMALL_INT(type); + t->items[2] = MP_OBJ_NEW_SMALL_INT(0); + t->items[3] = mp_obj_new_int(data_len); + + return MP_OBJ_FROM_PTR(t); + } + + return MP_OBJ_STOP_ITERATION; +} + +static mp_obj_t vfs_rom_ilistdir(mp_obj_t self_in, mp_obj_t path_in) { + mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in); + vfs_rom_ilistdir_it_t *iter = m_new_obj(vfs_rom_ilistdir_it_t); + iter->base.type = &mp_type_polymorph_iter; + iter->iternext = vfs_rom_ilistdir_it_iternext; + iter->vfs_rom = self; + iter->is_str = mp_obj_get_type(path_in) == &mp_type_str; + const char *path = mp_vfs_rom_get_path_str(self, path_in); + size_t size; + if (mp_vfs_rom_search_filesystem(self, path, &size, &iter->index) != MP_IMPORT_STAT_DIR) { + mp_raise_OSError(MP_ENOENT); + } + iter->index_top = iter->index + size; + return MP_OBJ_FROM_PTR(iter); +} +static MP_DEFINE_CONST_FUN_OBJ_2(vfs_rom_ilistdir_obj, vfs_rom_ilistdir); + +static mp_obj_t vfs_rom_stat(mp_obj_t self_in, mp_obj_t path_in) { + mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in); + const char *path = mp_vfs_rom_get_path_str(self, path_in); + size_t file_size; + const uint8_t *file_data; + mp_import_stat_t stat = mp_vfs_rom_search_filesystem(self, path, &file_size, &file_data); + if (stat == MP_IMPORT_STAT_NO_EXIST) { + mp_raise_OSError(MP_ENOENT); + } + mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL)); + t->items[0] = MP_OBJ_NEW_SMALL_INT(stat == MP_IMPORT_STAT_FILE ? MP_S_IFREG : MP_S_IFDIR); // st_mode + t->items[1] = MP_OBJ_NEW_SMALL_INT(0); // st_ino + t->items[2] = MP_OBJ_NEW_SMALL_INT(0); // st_dev + t->items[3] = MP_OBJ_NEW_SMALL_INT(0); // st_nlink + t->items[4] = MP_OBJ_NEW_SMALL_INT(0); // st_uid + t->items[5] = MP_OBJ_NEW_SMALL_INT(0); // st_gid + t->items[6] = MP_OBJ_NEW_SMALL_INT(file_size); // st_size + t->items[7] = MP_OBJ_NEW_SMALL_INT(0); // st_atime + t->items[8] = MP_OBJ_NEW_SMALL_INT(0); // st_mtime + t->items[9] = MP_OBJ_NEW_SMALL_INT(0); // st_ctime + return MP_OBJ_FROM_PTR(t); +} +static MP_DEFINE_CONST_FUN_OBJ_2(vfs_rom_stat_obj, vfs_rom_stat); + +static mp_obj_t vfs_rom_statvfs(mp_obj_t self_in, mp_obj_t path_in) { + mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in); + (void)path_in; + size_t filesystem_len = self->filesystem_end - self->filesystem; + mp_obj_tuple_t *t = MP_OBJ_TO_PTR(mp_obj_new_tuple(10, NULL)); + t->items[0] = MP_OBJ_NEW_SMALL_INT(1); // f_bsize + t->items[1] = MP_OBJ_NEW_SMALL_INT(0); // f_frsize + t->items[2] = mp_obj_new_int_from_uint(filesystem_len); // f_blocks + t->items[3] = MP_OBJ_NEW_SMALL_INT(0); // f_bfree + t->items[4] = MP_OBJ_NEW_SMALL_INT(0); // f_bavail + t->items[5] = MP_OBJ_NEW_SMALL_INT(0); // f_files + t->items[6] = MP_OBJ_NEW_SMALL_INT(0); // f_ffree + t->items[7] = MP_OBJ_NEW_SMALL_INT(0); // f_favail + t->items[8] = MP_OBJ_NEW_SMALL_INT(0); // f_flags + t->items[9] = MP_OBJ_NEW_SMALL_INT(32767); // f_namemax + return MP_OBJ_FROM_PTR(t); +} +static MP_DEFINE_CONST_FUN_OBJ_2(vfs_rom_statvfs_obj, vfs_rom_statvfs); + +static const mp_rom_map_elem_t vfs_rom_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_mount), MP_ROM_PTR(&vfs_rom_mount_obj) }, + { MP_ROM_QSTR(MP_QSTR_umount), MP_ROM_PTR(&mp_identity_obj) }, + { MP_ROM_QSTR(MP_QSTR_open), MP_ROM_PTR(&vfs_rom_open_obj) }, + + { MP_ROM_QSTR(MP_QSTR_chdir), MP_ROM_PTR(&vfs_rom_chdir_obj) }, + { MP_ROM_QSTR(MP_QSTR_ilistdir), MP_ROM_PTR(&vfs_rom_ilistdir_obj) }, + { MP_ROM_QSTR(MP_QSTR_stat), MP_ROM_PTR(&vfs_rom_stat_obj) }, + { MP_ROM_QSTR(MP_QSTR_statvfs), MP_ROM_PTR(&vfs_rom_statvfs_obj) }, +}; +static MP_DEFINE_CONST_DICT(vfs_rom_locals_dict, vfs_rom_locals_dict_table); + +static mp_import_stat_t mp_vfs_rom_import_stat(void *self_in, const char *path) { + mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in); + return mp_vfs_rom_search_filesystem(self, path, NULL, NULL); +} + +static const mp_vfs_proto_t vfs_rom_proto = { + .import_stat = mp_vfs_rom_import_stat, +}; + +MP_DEFINE_CONST_OBJ_TYPE( + mp_type_vfs_rom, + MP_QSTR_VfsRom, + MP_TYPE_FLAG_NONE, + make_new, vfs_rom_make_new, + protocol, &vfs_rom_proto, + locals_dict, &vfs_rom_locals_dict + ); + +#endif // MICROPY_VFS_ROM diff --git a/extmod/vfs_rom.h b/extmod/vfs_rom.h new file mode 100644 index 0000000000000..d8e2b911ba0ca --- /dev/null +++ b/extmod/vfs_rom.h @@ -0,0 +1,47 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2022 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_EXTMOD_VFS_ROM_H +#define MICROPY_INCLUDED_EXTMOD_VFS_ROM_H + +#include "py/builtin.h" +#include "py/obj.h" + +#if MICROPY_VFS_ROM + +typedef struct _mp_obj_vfs_rom_t mp_obj_vfs_rom_t; + +extern const mp_obj_type_t mp_type_vfs_rom; + +static inline const char *mp_vfs_rom_get_path_str(mp_obj_vfs_rom_t *self, mp_obj_t path) { + return mp_obj_str_get_str(path); +} + +mp_import_stat_t mp_vfs_rom_search_filesystem(mp_obj_vfs_rom_t *self, const char *path, size_t *size_out, const uint8_t **data_out); +mp_obj_t mp_vfs_rom_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_in); + +#endif // MICROPY_VFS_ROM + +#endif // MICROPY_INCLUDED_EXTMOD_VFS_ROM_H diff --git a/extmod/vfs_rom_file.c b/extmod/vfs_rom_file.c new file mode 100644 index 0000000000000..57aca8c5dc7d3 --- /dev/null +++ b/extmod/vfs_rom_file.c @@ -0,0 +1,180 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2022 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include + +#include "py/reader.h" +#include "py/runtime.h" +#include "py/stream.h" +#include "extmod/vfs_rom.h" + +#if MICROPY_VFS_ROM + +typedef struct _mp_obj_vfs_rom_file_t { + mp_obj_base_t base; + size_t file_size; + size_t file_offset; + const uint8_t *file_data; +} mp_obj_vfs_rom_file_t; + +static const mp_obj_type_t mp_type_vfs_rom_fileio; +static const mp_obj_type_t mp_type_vfs_rom_textio; + +mp_obj_t mp_vfs_rom_file_open(mp_obj_t self_in, mp_obj_t path_in, mp_obj_t mode_in) { + mp_obj_vfs_rom_t *self = MP_OBJ_TO_PTR(self_in); + + const char *mode_s = mp_obj_str_get_str(mode_in); + const mp_obj_type_t *type = &mp_type_vfs_rom_textio; + while (*mode_s) { + switch (*mode_s++) { + case 'r': + break; + case 'w': + case 'a': + case '+': + mp_raise_OSError(MP_EROFS); + case 'b': + type = &mp_type_vfs_rom_fileio; + break; + case 't': + type = &mp_type_vfs_rom_textio; + break; + } + } + + mp_obj_vfs_rom_file_t *o = m_new_obj(mp_obj_vfs_rom_file_t); + o->base.type = type; + o->file_offset = 0; + + const char *path = mp_vfs_rom_get_path_str(self, path_in); + mp_import_stat_t stat = mp_vfs_rom_search_filesystem(self, path, &o->file_size, &o->file_data); + if (stat == MP_IMPORT_STAT_NO_EXIST) { + mp_raise_OSError(MP_ENOENT); + } else if (stat == MP_IMPORT_STAT_DIR) { + mp_raise_OSError(MP_EISDIR); + } + + return MP_OBJ_FROM_PTR(o); +} + +static mp_int_t vfs_rom_file_get_buffer(mp_obj_t self_in, mp_buffer_info_t *bufinfo, mp_uint_t flags) { + mp_obj_vfs_rom_file_t *self = MP_OBJ_TO_PTR(self_in); + if (flags == MP_BUFFER_READ) { + bufinfo->buf = (void *)self->file_data; + bufinfo->len = self->file_size; + bufinfo->typecode = 'B'; + return 0; + } else { + // Can't write to a ROM file. + return 1; + } +} + +static mp_uint_t vfs_rom_file_read(mp_obj_t o_in, void *buf, mp_uint_t size, int *errcode) { + mp_obj_vfs_rom_file_t *self = MP_OBJ_TO_PTR(o_in); + size_t remain = self->file_size - self->file_offset; + if (size > remain) { + size = remain; + } + memcpy(buf, self->file_data + self->file_offset, size); + self->file_offset += size; + return size; +} + +static mp_uint_t vfs_rom_file_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, int *errcode) { + mp_obj_vfs_rom_file_t *self = MP_OBJ_TO_PTR(o_in); + + switch (request) { + case MP_STREAM_SEEK: { + struct mp_stream_seek_t *s = (struct mp_stream_seek_t *)arg; + if (s->whence == 0) { // SEEK_SET + self->file_offset = (size_t)s->offset; + } else if (s->whence == 1) { // SEEK_CUR + self->file_offset += s->offset; + } else { // SEEK_END + self->file_offset = self->file_size + s->offset; + } + if (self->file_offset > self->file_size) { + if (s->offset < 0) { + // Seek to before the start of the file. + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; + } + self->file_offset = self->file_size; + } + s->offset = self->file_offset; + return 0; + } + case MP_STREAM_CLOSE: + return 0; + default: + *errcode = MP_EINVAL; + return MP_STREAM_ERROR; + } +} + +static const mp_rom_map_elem_t vfs_rom_rawfile_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) }, + { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) }, + { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, + { MP_ROM_QSTR(MP_QSTR_readlines), MP_ROM_PTR(&mp_stream_unbuffered_readlines_obj) }, + { MP_ROM_QSTR(MP_QSTR_seek), MP_ROM_PTR(&mp_stream_seek_obj) }, + { MP_ROM_QSTR(MP_QSTR_tell), MP_ROM_PTR(&mp_stream_tell_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, + { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, + { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&mp_stream___exit___obj) }, +}; +static MP_DEFINE_CONST_DICT(vfs_rom_rawfile_locals_dict, vfs_rom_rawfile_locals_dict_table); + +static const mp_stream_p_t vfs_rom_fileio_stream_p = { + .read = vfs_rom_file_read, + .ioctl = vfs_rom_file_ioctl, +}; + +static MP_DEFINE_CONST_OBJ_TYPE( + mp_type_vfs_rom_fileio, + MP_QSTR_FileIO, + MP_TYPE_FLAG_ITER_IS_STREAM, + buffer, vfs_rom_file_get_buffer, + protocol, &vfs_rom_fileio_stream_p, + locals_dict, &vfs_rom_rawfile_locals_dict + ); + +static const mp_stream_p_t vfs_rom_textio_stream_p = { + .read = vfs_rom_file_read, + .ioctl = vfs_rom_file_ioctl, + .is_text = true, +}; + +static MP_DEFINE_CONST_OBJ_TYPE( + mp_type_vfs_rom_textio, + MP_QSTR_TextIOWrapper, + MP_TYPE_FLAG_ITER_IS_STREAM, + protocol, &vfs_rom_textio_stream_p, + locals_dict, &vfs_rom_rawfile_locals_dict + ); + +#endif // MICROPY_VFS_ROM diff --git a/ports/qemu/mpconfigport.h b/ports/qemu/mpconfigport.h index f95b58ba4d465..d3e3c0a3f37ac 100644 --- a/ports/qemu/mpconfigport.h +++ b/ports/qemu/mpconfigport.h @@ -60,6 +60,7 @@ #define MICROPY_PY_MACHINE_RESET (1) #define MICROPY_PY_MACHINE_PIN_BASE (1) #define MICROPY_VFS (1) +#define MICROPY_VFS_ROM (1) // type definitions for the specific machine diff --git a/ports/unix/variants/mpconfigvariant_common.h b/ports/unix/variants/mpconfigvariant_common.h index cea0397414325..093477af0aded 100644 --- a/ports/unix/variants/mpconfigvariant_common.h +++ b/ports/unix/variants/mpconfigvariant_common.h @@ -121,3 +121,5 @@ #define MICROPY_PY_MACHINE (1) #define MICROPY_PY_MACHINE_PULSE (1) #define MICROPY_PY_MACHINE_PIN_BASE (1) + +#define MICROPY_VFS_ROM (1) diff --git a/py/mpconfig.h b/py/mpconfig.h index a330cadc4ef3a..8598eaa5bb639 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -1016,6 +1016,11 @@ typedef double mp_float_t; #define MICROPY_VFS_LFS2 (0) #endif +// Support for ROMFS. +#ifndef MICROPY_VFS_ROM +#define MICROPY_VFS_ROM (0) +#endif + /*****************************************************************************/ /* Fine control over Python builtins, classes, modules, etc */ diff --git a/py/persistentcode.c b/py/persistentcode.c index be93eaa5b4564..0843d1a2c52e6 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -181,6 +181,15 @@ static qstr load_qstr(mp_reader_t *reader) { return len >> 1; } len >>= 1; + + #if MICROPY_VFS_ROM + // If possible, create the qstr from the memory-mapped string data. + const uint8_t *memmap = mp_reader_try_read_rom(reader, len + 1); + if (memmap != NULL) { + return qstr_from_strn_static((const char *)memmap, len); + } + #endif + char *str = m_new(char, len); read_bytes(reader, (byte *)str, len); read_byte(reader); // read and discard null terminator @@ -189,6 +198,24 @@ static qstr load_qstr(mp_reader_t *reader) { return qst; } +#if MICROPY_VFS_ROM +// Create a str/bytes object that can forever reference the given data. +static mp_obj_t mp_obj_new_str_static(const mp_obj_type_t *type, const byte *data, size_t len) { + if (type == &mp_type_str) { + qstr q = qstr_find_strn((const char *)data, len); + if (q != MP_QSTRnull) { + return MP_OBJ_NEW_QSTR(q); + } + } + assert(data[len] == '\0'); + mp_obj_str_t *o = mp_obj_malloc(mp_obj_str_t, type); + o->len = len; + o->hash = qstr_compute_hash(data, len); + o->data = data; + return MP_OBJ_FROM_PTR(o); +} +#endif + static mp_obj_t load_obj(mp_reader_t *reader) { byte obj_type = read_byte(reader); #if MICROPY_EMIT_MACHINE_CODE @@ -206,6 +233,8 @@ static mp_obj_t load_obj(mp_reader_t *reader) { return MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj); } else { size_t len = read_uint(reader); + + // Handle empty bytes object, and tuple objects. if (len == 0 && obj_type == MP_PERSISTENT_OBJ_BYTES) { read_byte(reader); // skip null terminator return mp_const_empty_bytes; @@ -216,11 +245,31 @@ static mp_obj_t load_obj(mp_reader_t *reader) { } return MP_OBJ_FROM_PTR(tuple); } + + // Read in the object's data, either from ROM or into RAM. + const uint8_t *memmap = NULL; vstr_t vstr; - vstr_init_len(&vstr, len); - read_bytes(reader, (byte *)vstr.buf, len); + #if MICROPY_VFS_ROM + memmap = mp_reader_try_read_rom(reader, len); + vstr.buf = (void *)memmap; + vstr.len = len; + #endif + if (memmap == NULL) { + // Data could not be memory-mapped, so allocate it in RAM and read it in. + vstr_init_len(&vstr, len); + read_bytes(reader, (byte *)vstr.buf, len); + } + + // Create and return the object. if (obj_type == MP_PERSISTENT_OBJ_STR || obj_type == MP_PERSISTENT_OBJ_BYTES) { - read_byte(reader); // skip null terminator + read_byte(reader); // skip null terminator (it needs to be there for ROM str objects) + #if MICROPY_VFS_ROM + if (memmap != NULL) { + // Create a str/bytes that references the memory-mapped data. + const mp_obj_type_t *t = obj_type == MP_PERSISTENT_OBJ_STR ? &mp_type_str : &mp_type_bytes; + return mp_obj_new_str_static(t, memmap, len); + } + #endif if (obj_type == MP_PERSISTENT_OBJ_STR) { return mp_obj_new_str_from_utf8_vstr(&vstr); } else { @@ -257,10 +306,17 @@ static mp_raw_code_t *load_raw_code(mp_reader_t *reader, mp_module_context_t *co #endif if (kind == MP_CODE_BYTECODE) { - // Allocate memory for the bytecode - fun_data = m_new(uint8_t, fun_data_len); - // Load bytecode - read_bytes(reader, fun_data, fun_data_len); + #if MICROPY_VFS_ROM + // Try to reference memory-mapped data for the bytecode. + fun_data = (uint8_t *)mp_reader_try_read_rom(reader, fun_data_len); + #endif + + if (fun_data == NULL) { + // Allocate memory for the bytecode. + fun_data = m_new(uint8_t, fun_data_len); + // Load bytecode. + read_bytes(reader, fun_data, fun_data_len); + } #if MICROPY_EMIT_MACHINE_CODE } else { diff --git a/py/qstr.c b/py/qstr.c index fea7f44fe13bf..7902646a92dc5 100644 --- a/py/qstr.c +++ b/py/qstr.c @@ -309,7 +309,7 @@ qstr qstr_from_str(const char *str) { return qstr_from_strn(str, strlen(str)); } -qstr qstr_from_strn(const char *str, size_t len) { +static qstr qstr_from_strn_helper(const char *str, size_t len, bool data_is_static) { QSTR_ENTER(); qstr q = qstr_find_strn(str, len); if (q == 0) { @@ -321,6 +321,12 @@ qstr qstr_from_strn(const char *str, size_t len) { mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("name too long")); } + if (data_is_static) { + // Given string data will be forever available so use it directly. + assert(str[len] == '\0'); + goto add; + } + // compute number of bytes needed to intern this string size_t n_bytes = len + 1; @@ -364,12 +370,26 @@ qstr qstr_from_strn(const char *str, size_t len) { // store the interned strings' data memcpy(q_ptr, str, len); q_ptr[len] = '\0'; - q = qstr_add(len, q_ptr); + str = q_ptr; + + add: + q = qstr_add(len, str); } QSTR_EXIT(); return q; } +qstr qstr_from_strn(const char *str, size_t len) { + return qstr_from_strn_helper(str, len, false); +} + +#if MICROPY_VFS_ROM +// Create a new qstr that can forever reference the given string data. +qstr qstr_from_strn_static(const char *str, size_t len) { + return qstr_from_strn_helper(str, len, true); +} +#endif + mp_uint_t qstr_hash(qstr q) { const qstr_pool_t *pool = find_qstr(&q); #if MICROPY_QSTR_BYTES_IN_HASH diff --git a/py/qstr.h b/py/qstr.h index 58ce285fd2c8a..0cde6062eb1a5 100644 --- a/py/qstr.h +++ b/py/qstr.h @@ -101,6 +101,9 @@ qstr qstr_find_strn(const char *str, size_t str_len); // returns MP_QSTRnull if qstr qstr_from_str(const char *str); qstr qstr_from_strn(const char *str, size_t len); +#if MICROPY_VFS_ROM +qstr qstr_from_strn_static(const char *str, size_t len); +#endif mp_uint_t qstr_hash(qstr q); const char *qstr_str(qstr q); diff --git a/py/reader.c b/py/reader.c index 151e04cac2c71..8feb6d75275f1 100644 --- a/py/reader.c +++ b/py/reader.c @@ -50,7 +50,7 @@ static mp_uint_t mp_reader_mem_readbyte(void *data) { static void mp_reader_mem_close(void *data) { mp_reader_mem_t *reader = (mp_reader_mem_t *)data; - if (reader->free_len > 0) { + if (reader->free_len > 0 && reader->free_len != MP_READER_IS_ROM) { m_del(char, (char *)reader->beg, reader->free_len); } m_del_obj(mp_reader_mem_t, reader); @@ -67,6 +67,19 @@ void mp_reader_new_mem(mp_reader_t *reader, const byte *buf, size_t len, size_t reader->close = mp_reader_mem_close; } +const uint8_t *mp_reader_try_read_rom(mp_reader_t *reader, size_t len) { + if (reader->readbyte != mp_reader_mem_readbyte) { + return NULL; + } + mp_reader_mem_t *m = reader->data; + if (m->free_len != MP_READER_IS_ROM) { + return NULL; + } + const uint8_t *data = m->cur; + m->cur += len; + return data; +} + #if MICROPY_READER_POSIX #include diff --git a/py/reader.h b/py/reader.h index 5cb1e67966c61..301c70ab3f0d4 100644 --- a/py/reader.h +++ b/py/reader.h @@ -28,6 +28,10 @@ #include "py/obj.h" +// Pass to the `free_len` argument to `mp_reader_new_mem` to indicate that the data is in ROM. +// This means that the data is addressable and will remain valid at least until a soft reset. +#define MP_READER_IS_ROM ((size_t)-1) + // the readbyte function must return the next byte in the input stream // it must return MP_READER_EOF if end of stream // it can be called again after returning MP_READER_EOF, and in that case must return MP_READER_EOF @@ -43,4 +47,9 @@ void mp_reader_new_mem(mp_reader_t *reader, const byte *buf, size_t len, size_t void mp_reader_new_file(mp_reader_t *reader, qstr filename); void mp_reader_new_file_from_fd(mp_reader_t *reader, int fd, bool close_fd); +// Try to efficiently read the given number of bytes from a ROM-based reader. +// Returns a valid, non-NULL pointer to the requseted data if the reader points to ROM. +// Returns NULL if the reader does not point to ROM. +const uint8_t *mp_reader_try_read_rom(mp_reader_t *reader, size_t len); + #endif // MICROPY_INCLUDED_PY_READER_H diff --git a/tests/extmod/vfs_rom.py b/tests/extmod/vfs_rom.py new file mode 100644 index 0000000000000..0382c84c55894 --- /dev/null +++ b/tests/extmod/vfs_rom.py @@ -0,0 +1,412 @@ +# Test VfsRom filesystem. + +try: + import sys, struct, os, uctypes, vfs + + vfs.VfsRom +except (ImportError, AttributeError): + print("SKIP") + raise SystemExit + +try: + import select +except ImportError: + select = None + +import unittest + +IFDIR = 0x4000 +IFREG = 0x8000 + +SEEK_SET = 0 +SEEK_CUR = 1 +SEEK_END = 2 + +# An mpy file with four constant objects: str, bytes, long-int, float. +test_mpy = ( + # header + b"M\x06\x00\x1f" # mpy file header + b"\x06" # n_qstr + b"\x05" # n_obj + # qstrs + b"\x0etest.py\x00" # qstr0 = "test.py" + b"\x0f" # qstr1 = "" + b"\x0estr_obj\x00" # qstr2 = "str_obj" + b"\x12bytes_obj\x00" # qstr3 = "bytes_obj" + b"\x0eint_obj\x00" # qstr4 = "int_obj" + b"\x12float_obj\x00" # qstr5 = "float_obj" + # objects + b"\x05\x14this is a str object\x00" + b"\x06\x16this is a bytes object\x00" + b"\x07\x0a1234567890" # long-int object + b"\x08\x041.23" # float object + b"\x05\x07str_obj\x00" # str object of existing qstr + # bytecode + b"\x81\x28" # 21 bytes, no children, bytecode + b"\x00\x02" # prelude + b"\x01" # simple name () + b"\x23\x00" # LOAD_CONST_OBJ(0) + b"\x16\x02" # STORE_NAME(str_obj) + b"\x23\x01" # LOAD_CONST_OBJ(1) + b"\x16\x03" # STORE_NAME(bytes_obj) + b"\x23\x02" # LOAD_CONST_OBJ(2) + b"\x16\x04" # STORE_NAME(int_obj) + b"\x23\x03" # LOAD_CONST_OBJ(3) + b"\x16\x05" # STORE_NAME(float_obj) + b"\x51" # LOAD_CONST_NONE + b"\x63" # RETURN_VALUE +) + + +class VfsRomWriter: + ROMFS_HEADER = b"\xd2\xcd\x31" + + ROMFS_RECORD_KIND_UNUSED = 0 + ROMFS_RECORD_KIND_PADDING = 1 + ROMFS_RECORD_KIND_DATA_VERBATIM = 2 + ROMFS_RECORD_KIND_DATA_POINTER = 3 + ROMFS_RECORD_KIND_DIRECTORY = 4 + ROMFS_RECORD_KIND_FILE = 5 + + def __init__(self): + self._dir_stack = [(None, bytearray())] + + def _encode_uint(self, value): + encoded = [value & 0x7F] + value >>= 7 + while value != 0: + encoded.insert(0, 0x80 | (value & 0x7F)) + value >>= 7 + return bytes(encoded) + + def _pack(self, kind, payload): + return self._encode_uint(kind) + self._encode_uint(len(payload)) + payload + + def _extend(self, data): + buf = self._dir_stack[-1][1] + buf.extend(data) + return len(buf) + + def finalise(self): + _, data = self._dir_stack.pop() + encoded_kind = VfsRomWriter.ROMFS_HEADER + encoded_len = self._encode_uint(len(data)) + if (len(encoded_kind) + len(encoded_len) + len(data)) % 2 == 1: + encoded_len = b"\x80" + encoded_len + data = encoded_kind + encoded_len + data + return data + + def opendir(self, dirname): + self._dir_stack.append((dirname, bytearray())) + + def closedir(self): + dirname, dirdata = self._dir_stack.pop() + dirdata = self._encode_uint(len(dirname)) + bytes(dirname, "ascii") + dirdata + self._extend(self._pack(VfsRomWriter.ROMFS_RECORD_KIND_DIRECTORY, dirdata)) + + def mkdata(self, data): + assert len(self._dir_stack) == 1 + return self._extend(self._pack(VfsRomWriter.ROMFS_RECORD_KIND_DATA_VERBATIM, data)) - len( + data + ) + + def mkfile(self, filename, filedata, extra_payload=b""): + filename = bytes(filename, "ascii") + payload = self._encode_uint(len(filename)) + payload += filename + payload += extra_payload + if isinstance(filedata, tuple): + sub_payload = self._encode_uint(filedata[0]) + sub_payload += self._encode_uint(filedata[1]) + payload += self._pack(VfsRomWriter.ROMFS_RECORD_KIND_DATA_POINTER, sub_payload) + else: + payload += self._pack(VfsRomWriter.ROMFS_RECORD_KIND_DATA_VERBATIM, filedata) + self._dir_stack[-1][1].extend(self._pack(VfsRomWriter.ROMFS_RECORD_KIND_FILE, payload)) + + +def _make_romfs(fs, files, data_map): + for filename, contents in files: + if isinstance(contents, tuple): + fs.opendir(filename) + _make_romfs(fs, contents, data_map) + fs.closedir() + elif isinstance(contents, int): + fs.mkfile(filename, data_map[contents]) + else: + fs.mkfile(filename, contents) + + +def make_romfs(files, data=None): + fs = VfsRomWriter() + data_map = {} + if data: + for k, v in data.items(): + data_map[k] = len(v), fs.mkdata(v) + _make_romfs(fs, files, data_map) + return fs.finalise() + + +# A class to test if a value is within a range, needed because MicroPython's range +# doesn't support arbitrary objects. +class Range: + def __init__(self, lower, upper): + self._lower = lower + self._upper = upper + + def __repr__(self): + return "Range({}, {})".format(self._lower, self._upper) + + def __contains__(self, value): + return self._lower <= value < self._upper + + +class TestBase(unittest.TestCase): + @classmethod + def setUpClass(cls): + fs_inner = make_romfs((("test_inner.txt", b"contents_inner"), ("c.py", b""))) + cls.romfs = make_romfs( + ( + ("fs.romfs", 0), + ("test.txt", b"contents"), + ( + "dir", + ( + ("a.py", b"x = 1"), + ("b.py", b"x = 2"), + ("test.mpy", test_mpy), + ), + ), + ), + {0: fs_inner}, + ) + cls.romfs_ilistdir = [ + ("fs.romfs", IFREG, 0, 46), + ("test.txt", IFREG, 0, 8), + ("dir", IFDIR, 0, 198), + ] + cls.romfs_listdir = [x[0] for x in cls.romfs_ilistdir] + cls.romfs_listdir_dir = ["a.py", "b.py", "test.mpy"] + cls.romfs_listdir_bytes = [bytes(x, "ascii") for x in cls.romfs_listdir] + cls.romfs_addr = uctypes.addressof(cls.romfs) + cls.romfs_addr_range = Range(cls.romfs_addr, cls.romfs_addr + len(cls.romfs)) + + +class TestEdgeCases(unittest.TestCase): + def test_empty_romfs(self): + empty_romfs = make_romfs(()) + self.assertEqual(empty_romfs, bytes([0x80 | ord("R"), 0x80 | ord("M"), ord("1"), 0])) + fs = vfs.VfsRom(empty_romfs) + self.assertIsInstance(fs, vfs.VfsRom) + fs.mount(True, False) + self.assertEqual(list(fs.ilistdir("")), []) + self.assertEqual(fs.stat(""), (IFDIR, 0, 0, 0, 0, 0, 0, 0, 0, 0)) + self.assertEqual(fs.statvfs(""), (1, 0, 0, 0, 0, 0, 0, 0, 0, 32767)) + + def test_error(self): + for bad_romfs in (b"", b"xxx", b"not a romfs"): + with self.assertRaises(OSError) as ctx: + vfs.VfsRom(bad_romfs) + self.assertEqual(ctx.exception.errno, errno.ENODEV) + + def test_unknown_record(self): + fs = VfsRomWriter() + fs._extend(fs._pack(VfsRomWriter.ROMFS_RECORD_KIND_PADDING, b"payload")) + fs.mkfile( + "test", + b"contents", + extra_payload=fs._pack(VfsRomWriter.ROMFS_RECORD_KIND_PADDING, b"pad"), + ) + romfs = fs.finalise() + fs = vfs.VfsRom(romfs) + self.assertEqual(list(fs.ilistdir("")), [("test", IFREG, 0, 8)]) + with fs.open("test", "rb") as f: + self.assertEqual(f.read(), b"contents") + + +class TestStandalone(TestBase): + def test_constructor(self): + self.assertIsInstance(vfs.VfsRom(self.romfs), vfs.VfsRom) + self.assertIsInstance(vfs.VfsRom(self.romfs_addr), vfs.VfsRom) + + def test_mount(self): + vfs.VfsRom(self.romfs).mount(True, False) + with self.assertRaises(OSError): + vfs.VfsRom(self.romfs).mount(True, True) + + def test_ilistdir(self): + fs = vfs.VfsRom(self.romfs) + self.assertEqual(list(fs.ilistdir("")), self.romfs_ilistdir) + self.assertEqual(list(fs.ilistdir("/")), self.romfs_ilistdir) + with self.assertRaises(OSError): + fs.ilistdir("does not exist") + + def test_stat(self): + fs = vfs.VfsRom(self.romfs) + self.assertEqual(fs.stat(""), (IFDIR, 0, 0, 0, 0, 0, 289, 0, 0, 0)) + self.assertEqual(fs.stat("/"), (IFDIR, 0, 0, 0, 0, 0, 289, 0, 0, 0)) + self.assertEqual(fs.stat("/test.txt"), (IFREG, 0, 0, 0, 0, 0, 8, 0, 0, 0)) + self.assertEqual(fs.stat("/dir"), (IFDIR, 0, 0, 0, 0, 0, 198, 0, 0, 0)) + with self.assertRaises(OSError): + fs.stat("/does-not-exist") + + def test_statvfs(self): + fs = vfs.VfsRom(self.romfs) + self.assertEqual(fs.statvfs(""), (1, 0, 289, 0, 0, 0, 0, 0, 0, 32767)) + + def test_open(self): + fs = vfs.VfsRom(self.romfs) + + with fs.open("/test.txt", "") as f: + self.assertEqual(f.read(), "contents") + with fs.open("/test.txt", "rt") as f: + self.assertEqual(f.read(), "contents") + with fs.open("/test.txt", "rb") as f: + self.assertEqual(f.read(), b"contents") + + with self.assertRaises(OSError) as ctx: + fs.open("/file-does-not-exist", "") + self.assertEqual(ctx.exception.errno, errno.ENOENT) + + with self.assertRaises(OSError) as ctx: + fs.open("/dir", "rb") + self.assertEqual(ctx.exception.errno, errno.EISDIR) + + with self.assertRaises(OSError): + fs.open("/test.txt", "w") + with self.assertRaises(OSError): + fs.open("/test.txt", "a") + with self.assertRaises(OSError): + fs.open("/test.txt", "+") + + def test_file_seek(self): + fs = vfs.VfsRom(self.romfs) + with fs.open("/test.txt", "") as f: + self.assertEqual(f.seek(0, SEEK_SET), 0) + self.assertEqual(f.seek(3, SEEK_SET), 3) + self.assertEqual(f.read(), "tents") + self.assertEqual(f.seek(0, SEEK_SET), 0) + self.assertEqual(f.seek(100, SEEK_CUR), 8) + self.assertEqual(f.seek(-1, SEEK_END), 7) + self.assertEqual(f.read(), "s") + self.assertEqual(f.seek(1, SEEK_END), 8) + with self.assertRaises(OSError): + f.seek(-1, SEEK_SET) + f.seek(0, SEEK_SET) + with self.assertRaises(OSError): + f.seek(-1, SEEK_CUR) + with self.assertRaises(OSError): + f.seek(-100, SEEK_END) + + @unittest.skipIf(select is None, "no select module") + def test_file_ioctl_invalid(self): + fs = vfs.VfsRom(self.romfs) + with fs.open("/test.txt", "") as f: + p = select.poll() + p.register(f) + with self.assertRaises(OSError): + p.poll(0) + + def test_memory_mapping(self): + fs = vfs.VfsRom(self.romfs) + with fs.open("/test.txt", "rb") as f: + addr = uctypes.addressof(f) + data = memoryview(f) + self.assertIn(addr, self.romfs_addr_range) + self.assertIn(addr + len(data), self.romfs_addr_range) + self.assertEqual(bytes(data), b"contents") + + +class TestMounted(TestBase): + def setUp(self): + self.orig_sys_path = list(sys.path) + self.orig_cwd = os.getcwd() + vfs.mount(vfs.VfsRom(self.romfs), "/test_rom") + + def tearDown(self): + vfs.umount("/test_rom") + os.chdir(self.orig_cwd) + sys.path = self.orig_sys_path + + def test_listdir(self): + self.assertEqual(os.listdir("/test_rom"), self.romfs_listdir) + self.assertEqual(os.listdir("/test_rom/dir"), self.romfs_listdir_dir) + self.assertEqual(os.listdir(b"/test_rom"), self.romfs_listdir_bytes) + + def test_chdir(self): + os.chdir("/test_rom") + self.assertEqual(os.listdir(), self.romfs_listdir) + + os.chdir("/test_rom/") + self.assertEqual(os.listdir(), self.romfs_listdir) + + # chdir within the romfs is not implemented. + with self.assertRaises(OSError): + os.chdir("/test_rom/dir") + + def test_stat(self): + self.assertEqual(os.stat("/test_rom"), (IFDIR, 0, 0, 0, 0, 0, 289, 0, 0, 0)) + self.assertEqual(os.stat("/test_rom/"), (IFDIR, 0, 0, 0, 0, 0, 289, 0, 0, 0)) + self.assertEqual(os.stat("/test_rom/test.txt"), (IFREG, 0, 0, 0, 0, 0, 8, 0, 0, 0)) + self.assertEqual(os.stat("/test_rom/dir"), (IFDIR, 0, 0, 0, 0, 0, 198, 0, 0, 0)) + with self.assertRaises(OSError): + os.stat("/test_rom/does-not-exist") + + def test_open(self): + with open("/test_rom/test.txt") as f: + self.assertEqual(f.read(), "contents") + with open("/test_rom/test.txt", "b") as f: + self.assertEqual(f.read(), b"contents") + + with self.assertRaises(OSError) as ctx: + open("/test_rom/file-does-not-exist") + self.assertEqual(ctx.exception.errno, errno.ENOENT) + + with self.assertRaises(OSError) as ctx: + open("/test_rom/dir") + self.assertEqual(ctx.exception.errno, errno.EISDIR) + + def test_import_py(self): + sys.path.append("/test_rom/dir") + a = __import__("a") + b = __import__("b") + self.assertEqual(a.__file__, "/test_rom/dir/a.py") + self.assertEqual(a.x, 1) + self.assertEqual(b.__file__, "/test_rom/dir/b.py") + self.assertEqual(b.x, 2) + + def test_import_mpy(self): + sys.path.append("/test_rom/dir") + test = __import__("test") + self.assertEqual(test.__file__, "/test_rom/dir/test.mpy") + self.assertEqual(test.str_obj, "this is a str object") + self.assertEqual(test.bytes_obj, b"this is a bytes object") + self.assertEqual(test.int_obj, 1234567890) + self.assertEqual(test.float_obj, 1.23) + self.assertIn(uctypes.addressof(test.str_obj), self.romfs_addr_range) + self.assertIn(uctypes.addressof(test.bytes_obj), self.romfs_addr_range) + + def test_romfs_inner(self): + with open("/test_rom/fs.romfs", "rb") as f: + romfs_inner = vfs.VfsRom(memoryview(f)) + + vfs.mount(romfs_inner, "/test_rom2") + + self.assertEqual(os.listdir("/test_rom2"), ["test_inner.txt", "c.py"]) + + sys.path.append("/test_rom2") + self.assertEqual(__import__("c").__file__, "/test_rom2/c.py") + + with open("/test_rom2/test_inner.txt") as f: + self.assertEqual(f.read(), "contents_inner") + + with open("/test_rom2/test_inner.txt", "rb") as f: + addr = uctypes.addressof(f) + data = memoryview(f) + self.assertIn(addr, self.romfs_addr_range) + self.assertIn(addr + len(data), self.romfs_addr_range) + + vfs.umount("/test_rom2") + + +if __name__ == "__main__": + unittest.main()