Skip to content

mpremote: Support bytecode raw paste for 'mpremote run module.mpy' #8744

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

mpremote: Support bytecode raw paste for 'mpremote run module.mpy' #8744

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8639238
mpremote: Support bytecode raw paste for 'mpremote run module.mpy'.
projectgus Jun 9, 2022
9538230
mpremote: Allow 'run .mpy' for devices which don't support raw paste.
projectgus Jun 16, 2022
54e5d29
pyboard: Clean up if import fails when injecting mpy in a variable.
projectgus Jun 16, 2022
02826f0
pyexec: Add named constants for raw repl control characters.
projectgus Jun 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 72 additions & 26 deletions shared/runtime/pyexec.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
#include "py/gc.h"
#include "py/frozenmod.h"
#include "py/mphal.h"
#include "py/persistentcode.h"
#if MICROPY_HW_ENABLE_USB
#include "irq.h"
#include "usb.h"
Expand All @@ -58,6 +59,24 @@ STATIC bool repl_display_debugging_info = 0;
#define EXEC_FLAG_SOURCE_IS_FILENAME (1 << 5)
#define EXEC_FLAG_SOURCE_IS_READER (1 << 6)

#define RAWCODE_PASTE_NUM_ESCAPED 8 // This value has to match the same constant in tools/pyboard.py

// Raw REPL serial protocol control sequences
#define RAW_REPL_CTRL_INIT CHAR_CTRL_A
#define RAW_REPL_CTRL_EXIT_TO_FRIENDLY CHAR_CTRL_B
#define RAW_REPL_CTRL_CLEAR_LINE CHAR_CTRL_C
#define RAW_REPL_CTRL_EOF CHAR_CTRL_D
#define RAW_REPL_CTRL_INIT_CMD CHAR_CTRL_E
// CHAR_CTRL_F is recognised in raw paste mode (as an escape sequence), but not in raw REPL mode

// Sequence ^A ^E (RAW_REPL_CTRL_INIT, RAW_REPL_CTRL_INIT_CMD) can initiate one or more "init commands" based on the next
// character in the sequence:
#define RAW_REPL_INIT_CMD_PASTE_SOURCE 'A'
#define RAW_REPL_INIT_CMD_PASTE_RAWCODE 'B'

#define RAW_REPL_INIT_CMD_RESP_UNSUPPORTED "R\x00"
#define RAW_REPL_INIT_CMD_RESP_OK "R\x01"

// parses, compiles and executes the code in the lexer
// frees the lexer before returning
// EXEC_FLAG_PRINT_EOF prints 2 EOF chars: 1 after normal output, 1 after exception output
Expand All @@ -80,16 +99,26 @@ STATIC int parse_compile_execute(const void *source, mp_parse_input_kind_t input
nlr.ret_val = NULL;
if (nlr_push(&nlr) == 0) {
mp_obj_t module_fun;
#if MICROPY_MODULE_FROZEN_MPY
#if MICROPY_MODULE_FROZEN_MPY || MICROPY_PERSISTENT_CODE_LOAD
if (exec_flags & EXEC_FLAG_SOURCE_IS_RAW_CODE) {
// source is a raw_code object, create the function
const mp_frozen_module_t *frozen = source;
mp_module_context_t *ctx = m_new_obj(mp_module_context_t);
ctx->module.globals = mp_globals_get();
ctx->constants = frozen->constants;
module_fun = mp_make_function_from_raw_code(frozen->rc, ctx, NULL);

#if MICROPY_PERSISTENT_CODE_LOAD
if (exec_flags & EXEC_FLAG_SOURCE_IS_READER) {
// source is a reader that will give us raw code (mpy file equivalent)
mp_compiled_module_t cm = mp_raw_code_load((mp_reader_t *)source, ctx);
module_fun = mp_make_function_from_raw_code(cm.rc, ctx, NULL);
} else
#endif // MICROPY_PERSISTENT_CODE_LOAD
{
// source is a raw_code object, create the module function from it
const mp_frozen_module_t *frozen = source;
ctx->constants = frozen->constants;
module_fun = mp_make_function_from_raw_code(frozen->rc, ctx, NULL);
}
} else
#endif
#endif // MICROPY_MODULE_FROZEN_PY || MICROPY_PERSISTENT_CODE_LOAD
{
#if MICROPY_ENABLE_COMPILER
mp_lexer_t *lex;
Expand All @@ -109,7 +138,7 @@ STATIC int parse_compile_execute(const void *source, mp_parse_input_kind_t input
module_fun = mp_compile(&parse_tree, source_name, exec_flags & EXEC_FLAG_IS_REPL);
#else
mp_raise_msg(&mp_type_RuntimeError, MP_ERROR_TEXT("script compilation not supported"));
#endif
#endif // MICROPY_ENABLE_COMPILER
}

// execute code
Expand Down Expand Up @@ -220,6 +249,10 @@ STATIC mp_uint_t mp_reader_stdin_readbyte(void *data) {
} else {
return MP_READER_EOF;
}
} else if (c == CHAR_CTRL_F) {
// escape sequence, next character is escaped by adding RAWCODE_PASTE_NUM_ESCAPED to it
int e = mp_hal_stdin_rx_chr();
c = e - RAWCODE_PASTE_NUM_ESCAPED;
}

if (--reader->window_remain == 0) {
Expand Down Expand Up @@ -260,20 +293,33 @@ STATIC void mp_reader_new_stdin(mp_reader_t *reader, mp_reader_stdin_t *reader_s
reader->close = mp_reader_stdin_close;
}

STATIC int do_reader_stdin(int c) {
if (c != 'A') {
STATIC int handle_raw_repl_init_cmd(int c) {
int exec_flags = EXEC_FLAG_PRINT_EOF | EXEC_FLAG_SOURCE_IS_READER;
bool supported_command = false;

if (c == RAW_REPL_INIT_CMD_PASTE_SOURCE) {
supported_command = true;
}
#if MICROPY_PERSISTENT_CODE_LOAD
if (c == RAW_REPL_INIT_CMD_PASTE_RAWCODE) {
exec_flags |= EXEC_FLAG_SOURCE_IS_RAW_CODE;
supported_command = true;
}
#endif

if (!supported_command) {
// Unsupported command.
mp_hal_stdout_tx_strn("R\x00", 2);
mp_hal_stdout_tx_strn(RAW_REPL_INIT_CMD_RESP_UNSUPPORTED, 2);
return 0;
}

// Indicate reception of command.
mp_hal_stdout_tx_strn("R\x01", 2);
mp_hal_stdout_tx_strn(RAW_REPL_INIT_CMD_RESP_OK, 2);

// Entering raw paste mode
mp_reader_t reader;
mp_reader_stdin_t reader_stdin;
mp_reader_new_stdin(&reader, &reader_stdin, MICROPY_REPL_STDIN_BUFFER_MAX);
int exec_flags = EXEC_FLAG_PRINT_EOF | EXEC_FLAG_SOURCE_IS_READER;
return parse_compile_execute(&reader, MP_PARSE_FILE_INPUT, exec_flags);
}

Expand Down Expand Up @@ -308,30 +354,30 @@ void pyexec_event_repl_init(void) {
}

STATIC int pyexec_raw_repl_process_char(int c) {
if (c == CHAR_CTRL_A) {
if (c == RAW_REPL_CTRL_INIT) {
// reset raw REPL
if (vstr_len(MP_STATE_VM(repl_line)) == 2 && vstr_str(MP_STATE_VM(repl_line))[0] == CHAR_CTRL_E) {
int ret = do_reader_stdin(vstr_str(MP_STATE_VM(repl_line))[1]);
if (vstr_len(MP_STATE_VM(repl_line)) == 2 && vstr_str(MP_STATE_VM(repl_line))[0] == RAW_REPL_CTRL_INIT_CMD) {
int ret = handle_raw_repl_init_cmd(vstr_str(MP_STATE_VM(repl_line))[1]);
if (ret & PYEXEC_FORCED_EXIT) {
return ret;
}
goto reset;
}
mp_hal_stdout_tx_str("raw REPL; CTRL-B to exit\r\n");
goto reset;
} else if (c == CHAR_CTRL_B) {
} else if (c == RAW_REPL_CTRL_EXIT_TO_FRIENDLY) {
// change to friendly REPL
pyexec_mode_kind = PYEXEC_MODE_FRIENDLY_REPL;
vstr_reset(MP_STATE_VM(repl_line));
repl.cont_line = false;
repl.paste_mode = false;
pyexec_friendly_repl_process_char(CHAR_CTRL_B);
return 0;
} else if (c == CHAR_CTRL_C) {
} else if (c == RAW_REPL_CTRL_CLEAR_LINE) {
// clear line
vstr_reset(MP_STATE_VM(repl_line));
return 0;
} else if (c == CHAR_CTRL_D) {
} else if (c == RAW_REPL_CTRL_EOF) {
// input finished
} else {
// let through any other raw 8-bit value
Expand Down Expand Up @@ -392,7 +438,7 @@ STATIC int pyexec_friendly_repl_process_char(int c) {

if (!repl.cont_line) {

if (ret == CHAR_CTRL_A) {
if (ret == RAW_REPL_CTRL_INIT) {
// change to raw REPL
pyexec_mode_kind = PYEXEC_MODE_RAW_REPL;
mp_hal_stdout_tx_str("\r\n");
Expand Down Expand Up @@ -502,10 +548,10 @@ int pyexec_raw_repl(void) {
mp_hal_stdout_tx_str(">");
for (;;) {
int c = mp_hal_stdin_rx_chr();
if (c == CHAR_CTRL_A) {
if (c == RAW_REPL_CTRL_INIT) {
// reset raw REPL
if (vstr_len(&line) == 2 && vstr_str(&line)[0] == CHAR_CTRL_E) {
int ret = do_reader_stdin(vstr_str(&line)[1]);
if (vstr_len(&line) == 2 && vstr_str(&line)[0] == RAW_REPL_CTRL_INIT_CMD) {
int ret = handle_raw_repl_init_cmd(vstr_str(&line)[1]);
if (ret & PYEXEC_FORCED_EXIT) {
return ret;
}
Expand All @@ -514,16 +560,16 @@ int pyexec_raw_repl(void) {
continue;
}
goto raw_repl_reset;
} else if (c == CHAR_CTRL_B) {
} else if (c == RAW_REPL_CTRL_EXIT_TO_FRIENDLY) {
// change to friendly REPL
mp_hal_stdout_tx_str("\r\n");
vstr_clear(&line);
pyexec_mode_kind = PYEXEC_MODE_FRIENDLY_REPL;
return 0;
} else if (c == CHAR_CTRL_C) {
} else if (c == RAW_REPL_CTRL_CLEAR_LINE) {
// clear line
vstr_reset(&line);
} else if (c == CHAR_CTRL_D) {
} else if (c == RAW_REPL_CTRL_EOF) {
// input finished
break;
} else {
Expand Down Expand Up @@ -605,7 +651,7 @@ int pyexec_friendly_repl(void) {
int ret = readline(&line, mp_repl_get_ps1());
mp_parse_input_kind_t parse_input_kind = MP_PARSE_SINGLE_INPUT;

if (ret == CHAR_CTRL_A) {
if (ret == RAW_REPL_CTRL_INIT) {
// change to raw REPL
mp_hal_stdout_tx_str("\r\n");
vstr_clear(&line);
Expand Down
14 changes: 9 additions & 5 deletions tools/mpremote/mpremote/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -352,8 +352,10 @@ def do_repl_main_loop(pyb, console_in, console_out_write, *, code_to_inject, fil
pyb.enter_raw_repl(soft_reset=False)
with open(file_to_inject, "rb") as f:
pyfile = f.read()

is_bytecode = pyfile[0] == ord("M") and file_to_inject.endswith(".mpy")
try:
pyb.exec_raw_no_follow(pyfile)
pyb.exec_raw_no_follow(pyfile, is_bytecode)
except pyboard.PyboardError as er:
console_out_write(b"Error:\r\n")
console_out_write(er)
Expand Down Expand Up @@ -430,10 +432,10 @@ def console_out_write(b):
capture_file.close()


def execbuffer(pyb, buf, follow):
def execbuffer(pyb, buf, follow, is_bytecode=False):
ret_val = 0
try:
pyb.exec_raw_no_follow(buf)
pyb.exec_raw_no_follow(buf, is_bytecode)
if follow:
ret, ret_err = pyb.follow(timeout=None, data_consumer=pyboard.stdout_write_bytes)
if ret_err:
Expand Down Expand Up @@ -546,6 +548,7 @@ def main():
elif cmd == "umount":
pyb.umount_local()
elif cmd in ("exec", "eval", "run"):
is_bytecode = False
follow = True
if args[0] == "--no-follow":
args.pop(0)
Expand All @@ -554,15 +557,16 @@ def main():
buf = args.pop(0)
elif cmd == "eval":
buf = "print(" + args.pop(0) + ")"
else:
else: # run
filename = args.pop(0)
try:
with open(filename, "rb") as f:
buf = f.read()
is_bytecode = buf[0] == ord("M") and filename.endswith(".mpy")
except OSError:
print(f"{_PROG}: could not read file '{filename}'")
return 1
ret = execbuffer(pyb, buf, follow)
ret = execbuffer(pyb, buf, follow, is_bytecode)
if ret:
return ret
elif cmd == "fs":
Expand Down
42 changes: 34 additions & 8 deletions tools/pyboard.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
import sys
import time
import os
import re
import ast

try:
Expand Down Expand Up @@ -399,10 +400,22 @@ def raw_paste_write(self, command_bytes):
raise PyboardError("unexpected read during raw paste: {}".format(data))
# Send out as much data as possible that fits within the allowed window.
b = command_bytes[i : min(i + window_remain, len(command_bytes))]
self.serial.write(b)

window_remain -= len(b)
i += len(b)

# escape any characters that need to be escaped. Note this doesn't
# count towards the window size, as unescaping happens before filling
# the window buffer in the device
RAWCODE_PASTE_NUM_ESCAPED = 8 # value has to match the same constant in pyexec.c
b = re.sub(
rb"[" + bytes(range(RAWCODE_PASTE_NUM_ESCAPED)) + rb"]",
lambda c: bytes((0x06, c.group()[0] + RAWCODE_PASTE_NUM_ESCAPED)),
b,
)

self.serial.write(b)

# Indicate end of data.
self.serial.write(b"\x04")

Expand All @@ -411,7 +424,7 @@ def raw_paste_write(self, command_bytes):
if not data.endswith(b"\x04"):
raise PyboardError("could not complete raw paste: {}".format(data))

def exec_raw_no_follow(self, command):
def exec_raw_no_follow(self, command, is_bytecode=False):
if isinstance(command, bytes):
command_bytes = command
else:
Expand All @@ -424,7 +437,8 @@ def exec_raw_no_follow(self, command):

if self.use_raw_paste:
# Try to enter raw-paste mode.
self.serial.write(b"\x05A\x01")
raw_paste_cmd = b"\x05A\x01" if not is_bytecode else b"\x05B\x01"
self.serial.write(raw_paste_cmd)
data = self.serial.read(2)
if data == b"R\x00":
# Device understood raw-paste command but doesn't support it.
Expand All @@ -438,8 +452,18 @@ def exec_raw_no_follow(self, command):
if not data.endswith(b"w REPL; CTRL-B to exit\r\n>"):
print(data)
raise PyboardError("could not enter raw repl")
# Don't try to use raw-paste mode again for this connection.
self.use_raw_paste = False

if is_bytecode:
# if we can't raw paste bytecode then use the injected import hook to load it instead
command_bytes = "_injected_buf={!r}\n{}\n".format(
command_bytes, _injected_import_hook_code
)
if self.use_raw_paste and data == b"R\x00":
# the device did understand raw-paste, so try again as a plaintext raw paste
return self.exec_raw_no_follow(command_bytes, is_bytecode=False)

# Don't try to use raw-paste mode again for this connection.
self.use_raw_paste = False

# Write command using standard raw REPL, 256 bytes every 10ms.
for i in range(0, len(command_bytes), 256):
Expand Down Expand Up @@ -635,9 +659,11 @@ def open(self, path, mode):
return self.File()
uos.mount(_FS(), '/_')
uos.chdir('/_')
from _injected import *
uos.umount('/_')
del _injected_buf, _FS
try:
from _injected import *
finally:
uos.umount('/_')
del _injected_buf, _FS
"""


Expand Down