Skip to content

RFC py/stream: Add .readbin() and .writebin() methods. #3382

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

RFC py/stream: Add .readbin() and .writebin() methods. #3382

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ports/unix/file.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,7 @@ STATIC const mp_rom_map_elem_t rawfile_locals_dict_table[] = {
{ MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) },
{ MP_ROM_QSTR(MP_QSTR_readlines), MP_ROM_PTR(&mp_stream_unbuffered_readlines_obj) },
{ MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) },
{ MP_ROM_QSTR(MP_QSTR_writebin), MP_ROM_PTR(&mp_stream_writebin_obj) },
{ MP_ROM_QSTR(MP_QSTR_seek), MP_ROM_PTR(&mp_stream_seek_obj) },
{ MP_ROM_QSTR(MP_QSTR_tell), MP_ROM_PTR(&mp_stream_tell_obj) },
{ MP_ROM_QSTR(MP_QSTR_flush), MP_ROM_PTR(&mp_stream_flush_obj) },
Expand Down
5 changes: 5 additions & 0 deletions py/binary.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,4 +42,9 @@ void mp_binary_set_val(char struct_type, char val_type, mp_obj_t val_in, byte **
long long mp_binary_get_int(mp_uint_t size, bool is_signed, bool big_endian, const byte *src);
void mp_binary_set_int(mp_uint_t val_sz, bool big_endian, byte *dest, mp_uint_t val);

// Higher-level reusable functions from modstruct.c
char mp_struct_get_fmt_type(const char **fmt);
size_t mp_struct_calc_size_items(const char *fmt, size_t *total_sz);
void mp_struct_pack_into_internal(mp_obj_t fmt_in, byte *p, size_t n_args, const mp_obj_t *args);

#endif // MICROPY_INCLUDED_PY_BINARY_H
20 changes: 10 additions & 10 deletions py/modstruct.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
character data".
*/

STATIC char get_fmt_type(const char **fmt) {
char mp_struct_get_fmt_type(const char **fmt) {
char t = **fmt;
switch (t) {
case '!':
Expand Down Expand Up @@ -82,8 +82,8 @@ STATIC mp_uint_t get_fmt_num(const char **p) {
return val;
}

STATIC size_t calc_size_items(const char *fmt, size_t *total_sz) {
char fmt_type = get_fmt_type(&fmt);
size_t mp_struct_calc_size_items(const char *fmt, size_t *total_sz) {
char fmt_type = mp_struct_get_fmt_type(&fmt);
size_t total_cnt = 0;
size_t size;
for (size = 0; *fmt; fmt++) {
Expand Down Expand Up @@ -113,7 +113,7 @@ STATIC size_t calc_size_items(const char *fmt, size_t *total_sz) {
STATIC mp_obj_t struct_calcsize(mp_obj_t fmt_in) {
const char *fmt = mp_obj_str_get_str(fmt_in);
size_t size;
calc_size_items(fmt, &size);
mp_struct_calc_size_items(fmt, &size);
return MP_OBJ_NEW_SMALL_INT(size);
}
MP_DEFINE_CONST_FUN_OBJ_1(struct_calcsize_obj, struct_calcsize);
Expand All @@ -125,8 +125,8 @@ STATIC mp_obj_t struct_unpack_from(size_t n_args, const mp_obj_t *args) {
// we relax the "exact" requirement, and only implement "big enough".
const char *fmt = mp_obj_str_get_str(args[0]);
size_t total_sz;
size_t num_items = calc_size_items(fmt, &total_sz);
char fmt_type = get_fmt_type(&fmt);
size_t num_items = mp_struct_calc_size_items(fmt, &total_sz);
char fmt_type = mp_struct_get_fmt_type(&fmt);
mp_obj_tuple_t *res = MP_OBJ_TO_PTR(mp_obj_new_tuple(num_items, NULL));
mp_buffer_info_t bufinfo;
mp_get_buffer_raise(args[1], &bufinfo, MP_BUFFER_READ);
Expand Down Expand Up @@ -175,9 +175,9 @@ STATIC mp_obj_t struct_unpack_from(size_t n_args, const mp_obj_t *args) {
MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(struct_unpack_from_obj, 2, 3, struct_unpack_from);

// This function assumes there is enough room in p to store all the values
STATIC void struct_pack_into_internal(mp_obj_t fmt_in, byte *p, size_t n_args, const mp_obj_t *args) {
void mp_struct_pack_into_internal(mp_obj_t fmt_in, byte *p, size_t n_args, const mp_obj_t *args) {
const char *fmt = mp_obj_str_get_str(fmt_in);
char fmt_type = get_fmt_type(&fmt);
char fmt_type = mp_struct_get_fmt_type(&fmt);

size_t i;
for (i = 0; i < n_args;) {
Expand Down Expand Up @@ -217,7 +217,7 @@ STATIC mp_obj_t struct_pack(size_t n_args, const mp_obj_t *args) {
vstr_init_len(&vstr, size);
byte *p = (byte*)vstr.buf;
memset(p, 0, size);
struct_pack_into_internal(args[0], p, n_args - 1, &args[1]);
mp_struct_pack_into_internal(args[0], p, n_args - 1, &args[1]);
return mp_obj_new_str_from_vstr(&mp_type_bytes, &vstr);
}
MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(struct_pack_obj, 1, MP_OBJ_FUN_ARGS_MAX, struct_pack);
Expand All @@ -243,7 +243,7 @@ STATIC mp_obj_t struct_pack_into(size_t n_args, const mp_obj_t *args) {
mp_raise_ValueError("buffer too small");
}

struct_pack_into_internal(args[0], p, n_args - 3, &args[3]);
mp_struct_pack_into_internal(args[0], p, n_args - 3, &args[3]);
return mp_const_none;
}
MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(struct_pack_into_obj, 3, MP_OBJ_FUN_ARGS_MAX, struct_pack_into);
Expand Down
2 changes: 2 additions & 0 deletions py/objstringio.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,9 @@ STATIC const mp_rom_map_elem_t stringio_locals_dict_table[] = {
{ MP_ROM_QSTR(MP_QSTR_read), MP_ROM_PTR(&mp_stream_read_obj) },
{ MP_ROM_QSTR(MP_QSTR_readinto), MP_ROM_PTR(&mp_stream_readinto_obj) },
{ MP_ROM_QSTR(MP_QSTR_readline), MP_ROM_PTR(&mp_stream_unbuffered_readline_obj) },
{ MP_ROM_QSTR(MP_QSTR_readbin), MP_ROM_PTR(&mp_stream_readbin_obj) },
{ MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) },
{ MP_ROM_QSTR(MP_QSTR_writebin), MP_ROM_PTR(&mp_stream_writebin_obj) },
{ MP_ROM_QSTR(MP_QSTR_seek), MP_ROM_PTR(&mp_stream_seek_obj) },
{ MP_ROM_QSTR(MP_QSTR_flush), MP_ROM_PTR(&mp_stream_flush_obj) },
{ MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&stringio_close_obj) },
Expand Down
42 changes: 42 additions & 0 deletions py/stream.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
#include "py/objstr.h"
#include "py/stream.h"
#include "py/runtime.h"
#include "py/binary.h"

#if MICROPY_STREAMS_NON_BLOCK
#include <errno.h>
Expand Down Expand Up @@ -289,6 +290,21 @@ STATIC mp_obj_t stream_write1_method(mp_obj_t self_in, mp_obj_t arg) {
}
MP_DEFINE_CONST_FUN_OBJ_2(mp_stream_write1_obj, stream_write1_method);

STATIC mp_obj_t stream_writebin_method(mp_obj_t self_in, mp_obj_t fmt_in, mp_obj_t arg_in) {
size_t size;
const char *fmt = mp_obj_str_get_str(fmt_in);
size_t num = mp_struct_calc_size_items(fmt, &size);
if (num != 1) {
mp_raise_ValueError(NULL);
}

byte buf[size];
mp_struct_pack_into_internal(fmt_in, buf, 1, &arg_in);

return mp_stream_write(self_in, buf, size, MP_STREAM_RW_WRITE);
}
MP_DEFINE_CONST_FUN_OBJ_3(mp_stream_writebin_obj, stream_writebin_method);

STATIC mp_obj_t stream_readinto(size_t n_args, const mp_obj_t *args) {
mp_get_stream_raise(args[0], MP_STREAM_OP_READ);
mp_buffer_info_t bufinfo;
Expand Down Expand Up @@ -358,6 +374,32 @@ STATIC mp_obj_t stream_readall(mp_obj_t self_in) {
return mp_obj_new_str_from_vstr(STREAM_CONTENT_TYPE(stream_p), &vstr);
}

STATIC mp_obj_t stream_readbin(mp_obj_t self_in, mp_obj_t fmt_in) {
size_t size;
const char *fmt = mp_obj_str_get_str(fmt_in);
size_t num = mp_struct_calc_size_items(fmt, &size);
if (num != 1) {
mp_raise_ValueError(NULL);
}

byte buf[size];
int error;
mp_uint_t out_sz = mp_stream_read_exactly(self_in, buf, size, &error);

if (error != 0) {
mp_raise_OSError(error);
} else if (out_sz != size) {
nlr_raise(mp_obj_new_exception(&mp_type_EOFError));
}

byte *ptr = buf;
char fmt_type = mp_struct_get_fmt_type(&fmt);
mp_obj_t res = mp_binary_get_val(fmt_type, *fmt, &ptr);

return res;
}
MP_DEFINE_CONST_FUN_OBJ_2(mp_stream_readbin_obj, stream_readbin);

// Unbuffered, inefficient implementation of readline() for raw I/O files.
STATIC mp_obj_t stream_unbuffered_readline(size_t n_args, const mp_obj_t *args) {
const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_READ);
Expand Down
2 changes: 2 additions & 0 deletions py/stream.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,10 +65,12 @@ struct mp_stream_seek_t {
MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_read_obj);
MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_read1_obj);
MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_readinto_obj);
MP_DECLARE_CONST_FUN_OBJ_2(mp_stream_readbin_obj);
MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_unbuffered_readline_obj);
MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_unbuffered_readlines_obj);
MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_write_obj);
MP_DECLARE_CONST_FUN_OBJ_2(mp_stream_write1_obj);
MP_DECLARE_CONST_FUN_OBJ_3(mp_stream_writebin_obj);
MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_seek_obj);
MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_tell_obj);
MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_flush_obj);
Expand Down