From 858af77d6d3b918e2de98fdc66856b2d96253981 Mon Sep 17 00:00:00 2001 From: Paul Sokolovsky Date: Sat, 21 Oct 2017 14:46:56 +0300 Subject: [PATCH] py/stream: Add .readbin() and .writebin() methods. These methods allow read/write binary packed data directly from/to streams. That's quite a popular paradigm, and many programming languages/libraries use that (Java, Node.js, Android, etc.) Python doesn't support that natively, but such support would allow to implement parser for complex binary structures with low memory overhead, which would be beneficial for MicroPython. --- ports/unix/file.c | 1 + py/binary.h | 5 +++++ py/modstruct.c | 20 ++++++++++---------- py/objstringio.c | 2 ++ py/stream.c | 42 ++++++++++++++++++++++++++++++++++++++++++ py/stream.h | 2 ++ 6 files changed, 62 insertions(+), 10 deletions(-) diff --git a/ports/unix/file.c b/ports/unix/file.c index 84e9180821ce9..6689523cee0d8 100644 --- a/ports/unix/file.c +++ b/ports/unix/file.c @@ -221,6 +221,7 @@ STATIC const mp_rom_map_elem_t rawfile_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, { MP_ROM_QSTR(MP_QSTR_readlines), MP_ROM_PTR(&mp_stream_unbuffered_readlines_obj) }, { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) }, + { MP_ROM_QSTR(MP_QSTR_writebin), MP_ROM_PTR(&mp_stream_writebin_obj) }, { MP_ROM_QSTR(MP_QSTR_seek), MP_ROM_PTR(&mp_stream_seek_obj) }, { MP_ROM_QSTR(MP_QSTR_tell), MP_ROM_PTR(&mp_stream_tell_obj) }, { MP_ROM_QSTR(MP_QSTR_flush), MP_ROM_PTR(&mp_stream_flush_obj) }, diff --git a/py/binary.h b/py/binary.h index 0dae6a29e6617..4feccdff8639f 100644 --- a/py/binary.h +++ b/py/binary.h @@ -42,4 +42,9 @@ void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte ** long long mp_binary_get_int(mp_uint_t size, bool is_signed, bool big_endian, const byte *src); void mp_binary_set_int(mp_uint_t val_sz, bool big_endian, byte *dest, mp_uint_t val); +// Higher-level reusable functions from modstruct.c +char mp_struct_get_fmt_type(const char **fmt); +size_t mp_struct_calc_size_items(const char *fmt, size_t *total_sz); +void mp_struct_pack_into_internal(mp_obj_t fmt_in, byte *p, size_t n_args, const mp_obj_t *args); + #endif // MICROPY_INCLUDED_PY_BINARY_H diff --git a/py/modstruct.c b/py/modstruct.c index 8617a8e0d3e0d..8d3477a1621ba 100644 --- a/py/modstruct.c +++ b/py/modstruct.c @@ -52,7 +52,7 @@ character data". */ -STATIC char get_fmt_type(const char **fmt) { +char mp_struct_get_fmt_type(const char **fmt) { char t = **fmt; switch (t) { case '!': @@ -82,8 +82,8 @@ STATIC mp_uint_t get_fmt_num(const char **p) { return val; } -STATIC size_t calc_size_items(const char *fmt, size_t *total_sz) { - char fmt_type = get_fmt_type(&fmt); +size_t mp_struct_calc_size_items(const char *fmt, size_t *total_sz) { + char fmt_type = mp_struct_get_fmt_type(&fmt); size_t total_cnt = 0; size_t size; for (size = 0; *fmt; fmt++) { @@ -113,7 +113,7 @@ STATIC size_t calc_size_items(const char *fmt, size_t *total_sz) { STATIC mp_obj_t struct_calcsize(mp_obj_t fmt_in) { const char *fmt = mp_obj_str_get_str(fmt_in); size_t size; - calc_size_items(fmt, &size); + mp_struct_calc_size_items(fmt, &size); return MP_OBJ_NEW_SMALL_INT(size); } MP_DEFINE_CONST_FUN_OBJ_1(struct_calcsize_obj, struct_calcsize); @@ -125,8 +125,8 @@ STATIC mp_obj_t struct_unpack_from(size_t n_args, const mp_obj_t *args) { // we relax the "exact" requirement, and only implement "big enough". const char *fmt = mp_obj_str_get_str(args[0]); size_t total_sz; - size_t num_items = calc_size_items(fmt, &total_sz); - char fmt_type = get_fmt_type(&fmt); + size_t num_items = mp_struct_calc_size_items(fmt, &total_sz); + char fmt_type = mp_struct_get_fmt_type(&fmt); mp_obj_tuple_t *res = MP_OBJ_TO_PTR(mp_obj_new_tuple(num_items, NULL)); mp_buffer_info_t bufinfo; mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_READ); @@ -175,9 +175,9 @@ STATIC mp_obj_t struct_unpack_from(size_t n_args, const mp_obj_t *args) { MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(struct_unpack_from_obj, 2, 3, struct_unpack_from); // This function assumes there is enough room in p to store all the values -STATIC void struct_pack_into_internal(mp_obj_t fmt_in, byte *p, size_t n_args, const mp_obj_t *args) { +void mp_struct_pack_into_internal(mp_obj_t fmt_in, byte *p, size_t n_args, const mp_obj_t *args) { const char *fmt = mp_obj_str_get_str(fmt_in); - char fmt_type = get_fmt_type(&fmt); + char fmt_type = mp_struct_get_fmt_type(&fmt); size_t i; for (i = 0; i < n_args;) { @@ -217,7 +217,7 @@ STATIC mp_obj_t struct_pack(size_t n_args, const mp_obj_t *args) { vstr_init_len(&vstr, size); byte *p = (byte*)vstr.buf; memset(p, 0, size); - struct_pack_into_internal(args[0], p, n_args - 1, &args[1]); + mp_struct_pack_into_internal(args[0], p, n_args - 1, &args[1]); return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr); } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(struct_pack_obj, 1, MP_OBJ_FUN_ARGS_MAX, struct_pack); @@ -243,7 +243,7 @@ STATIC mp_obj_t struct_pack_into(size_t n_args, const mp_obj_t *args) { mp_raise_ValueError("buffer too small"); } - struct_pack_into_internal(args[0], p, n_args - 3, &args[3]); + mp_struct_pack_into_internal(args[0], p, n_args - 3, &args[3]); return mp_const_none; } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(struct_pack_into_obj, 3, MP_OBJ_FUN_ARGS_MAX, struct_pack_into); diff --git a/py/objstringio.c b/py/objstringio.c index 5c50aa31742ae..da369ecbb5536 100644 --- a/py/objstringio.c +++ b/py/objstringio.c @@ -230,7 +230,9 @@ STATIC const mp_rom_map_elem_t stringio_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) }, { MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) }, { MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) }, + { MP_ROM_QSTR(MP_QSTR_readbin), MP_ROM_PTR(&mp_stream_readbin_obj) }, { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) }, + { MP_ROM_QSTR(MP_QSTR_writebin), MP_ROM_PTR(&mp_stream_writebin_obj) }, { MP_ROM_QSTR(MP_QSTR_seek), MP_ROM_PTR(&mp_stream_seek_obj) }, { MP_ROM_QSTR(MP_QSTR_flush), MP_ROM_PTR(&mp_stream_flush_obj) }, { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&stringio_close_obj) }, diff --git a/py/stream.c b/py/stream.c index 453dee769fe8a..12504abc22549 100644 --- a/py/stream.c +++ b/py/stream.c @@ -31,6 +31,7 @@ #include "py/objstr.h" #include "py/stream.h" #include "py/runtime.h" +#include "py/binary.h" #if MICROPY_STREAMS_NON_BLOCK #include @@ -289,6 +290,21 @@ STATIC mp_obj_t stream_write1_method(mp_obj_t self_in, mp_obj_t arg) { } MP_DEFINE_CONST_FUN_OBJ_2(mp_stream_write1_obj, stream_write1_method); +STATIC mp_obj_t stream_writebin_method(mp_obj_t self_in, mp_obj_t fmt_in, mp_obj_t arg_in) { + size_t size; + const char *fmt = mp_obj_str_get_str(fmt_in); + size_t num = mp_struct_calc_size_items(fmt, &size); + if (num != 1) { + mp_raise_ValueError(NULL); + } + + byte buf[size]; + mp_struct_pack_into_internal(fmt_in, buf, 1, &arg_in); + + return mp_stream_write(self_in, buf, size, MP_STREAM_RW_WRITE); +} +MP_DEFINE_CONST_FUN_OBJ_3(mp_stream_writebin_obj, stream_writebin_method); + STATIC mp_obj_t stream_readinto(size_t n_args, const mp_obj_t *args) { mp_get_stream_raise(args[0], MP_STREAM_OP_READ); mp_buffer_info_t bufinfo; @@ -358,6 +374,32 @@ STATIC mp_obj_t stream_readall(mp_obj_t self_in) { return mp_obj_new_str_from_vstr(STREAM_CONTENT_TYPE(stream_p), &vstr); } +STATIC mp_obj_t stream_readbin(mp_obj_t self_in, mp_obj_t fmt_in) { + size_t size; + const char *fmt = mp_obj_str_get_str(fmt_in); + size_t num = mp_struct_calc_size_items(fmt, &size); + if (num != 1) { + mp_raise_ValueError(NULL); + } + + byte buf[size]; + int error; + mp_uint_t out_sz = mp_stream_read_exactly(self_in, buf, size, &error); + + if (error != 0) { + mp_raise_OSError(error); + } else if (out_sz != size) { + nlr_raise(mp_obj_new_exception(&mp_type_EOFError)); + } + + byte *ptr = buf; + char fmt_type = mp_struct_get_fmt_type(&fmt); + mp_obj_t res = mp_binary_get_val(fmt_type, *fmt, &ptr); + + return res; +} +MP_DEFINE_CONST_FUN_OBJ_2(mp_stream_readbin_obj, stream_readbin); + // Unbuffered, inefficient implementation of readline() for raw I/O files. STATIC mp_obj_t stream_unbuffered_readline(size_t n_args, const mp_obj_t *args) { const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_READ); diff --git a/py/stream.h b/py/stream.h index fbe3d7d859296..77b6d199c313b 100644 --- a/py/stream.h +++ b/py/stream.h @@ -65,10 +65,12 @@ struct mp_stream_seek_t { MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_read_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_read1_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_readinto_obj); +MP_DECLARE_CONST_FUN_OBJ_2(mp_stream_readbin_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_unbuffered_readline_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_unbuffered_readlines_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_write_obj); MP_DECLARE_CONST_FUN_OBJ_2(mp_stream_write1_obj); +MP_DECLARE_CONST_FUN_OBJ_3(mp_stream_writebin_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_seek_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_tell_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_flush_obj);