Skip to content

Commit 323ea3b

Browse files
aneeshdurgyouknowone
aneeshdurg
and
youknowone
authored
Support incomplete parsing (#5764)
* continue accepting REPL input for multiline strings * Match cpython behavior for all multi-line statements (execute when complete) * Emit _IncompleteInputError when compiling with incomplete flag * Refine when _IncompleteInputError is emitted * Support multiline strings emitting _IncompleteInputError * lint * Undo accidental change to PyTabError * match -> if let * Fix test_baseexception and test_codeop * fix spelling * fix exception name * Skip pickle test of _IncompleteInputError * Use py3.15's codeop implementation * Update Lib/test/test_baseexception.py --------- Co-authored-by: Jeong, YunWon <69878+youknowone@users.noreply.github.com>
1 parent e27d031 commit 323ea3b

File tree

10 files changed

+231
-38
lines changed

10 files changed

+231
-38
lines changed

‎Lib/codeop.py

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -65,14 +65,10 @@ def _maybe_compile(compiler, source, filename, symbol):
6565
try:
6666
compiler(source + "\n", filename, symbol)
6767
return None
68+
except _IncompleteInputError as e:
69+
return None
6870
except SyntaxError as e:
69-
# XXX: RustPython; support multiline definitions in REPL
70-
# See also: https://github.com/RustPython/RustPython/pull/5743
71-
strerr = str(e)
72-
if source.endswith(":") and "expected an indented block" in strerr:
73-
return None
74-
elif "incomplete input" in str(e):
75-
return None
71+
pass
7672
# fallthrough
7773

7874
return compiler(source, filename, symbol, incomplete_input=False)

‎Lib/test/test_baseexception.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,8 @@ def test_inheritance(self):
8383
exc_set = set(e for e in exc_set if not e.startswith('_'))
8484
# RUSTPYTHON specific
8585
exc_set.discard("JitError")
86+
# TODO: RUSTPYTHON; this will be officially introduced in Python 3.15
87+
exc_set.discard("IncompleteInputError")
8688
self.assertEqual(len(exc_set), 0, "%s not accounted for" % exc_set)
8789

8890
interface_tests = ("length", "args", "str", "repr")

‎Lib/test/test_pickle.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -664,6 +664,9 @@ def test_exceptions(self):
664664
BaseExceptionGroup,
665665
ExceptionGroup):
666666
continue
667+
# TODO: RUSTPYTHON: fix name mapping for _IncompleteInputError
668+
if exc is _IncompleteInputError:
669+
continue
667670
if exc is not OSError and issubclass(exc, OSError):
668671
self.assertEqual(reverse_mapping('builtins', name),
669672
('exceptions', 'OSError'))

‎compiler/src/lib.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ pub enum CompileErrorType {
2525
pub struct ParseError {
2626
#[source]
2727
pub error: parser::ParseErrorType,
28+
pub raw_location: ruff_text_size::TextRange,
2829
pub location: SourceLocation,
2930
pub source_path: String,
3031
}
@@ -48,6 +49,7 @@ impl CompileError {
4849
let location = source_code.source_location(error.location.start());
4950
Self::Parse(ParseError {
5051
error: error.error,
52+
raw_location: error.location,
5153
location,
5254
source_path: source_code.path.to_owned(),
5355
})

‎src/shell.rs

Lines changed: 75 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
mod helper;
22

33
use rustpython_compiler::{
4-
CompileError, ParseError, parser::LexicalErrorType, parser::ParseErrorType,
4+
CompileError, ParseError, parser::FStringErrorType, parser::LexicalErrorType,
5+
parser::ParseErrorType,
56
};
67
use rustpython_vm::{
78
AsObject, PyResult, VirtualMachine,
@@ -14,19 +15,26 @@ use rustpython_vm::{
1415
enum ShellExecResult {
1516
Ok,
1617
PyErr(PyBaseExceptionRef),
17-
Continue,
18+
ContinueBlock,
19+
ContinueLine,
1820
}
1921

2022
fn shell_exec(
2123
vm: &VirtualMachine,
2224
source: &str,
2325
scope: Scope,
2426
empty_line_given: bool,
25-
continuing: bool,
27+
continuing_block: bool,
2628
) -> ShellExecResult {
29+
// compiling expects only UNIX style line endings, and will replace windows line endings
30+
// internally. Since we might need to analyze the source to determine if an error could be
31+
// resolved by future input, we need the location from the error to match the source code that
32+
// was actually compiled.
33+
#[cfg(windows)]
34+
let source = &source.replace("\r\n", "\n");
2735
match vm.compile(source, compiler::Mode::Single, "<stdin>".to_owned()) {
2836
Ok(code) => {
29-
if empty_line_given || !continuing {
37+
if empty_line_given || !continuing_block {
3038
// We want to execute the full code
3139
match vm.run_code_obj(code, scope) {
3240
Ok(_val) => ShellExecResult::Ok,
@@ -40,8 +48,32 @@ fn shell_exec(
4048
Err(CompileError::Parse(ParseError {
4149
error: ParseErrorType::Lexical(LexicalErrorType::Eof),
4250
..
43-
})) => ShellExecResult::Continue,
51+
})) => ShellExecResult::ContinueLine,
52+
Err(CompileError::Parse(ParseError {
53+
error:
54+
ParseErrorType::Lexical(LexicalErrorType::FStringError(
55+
FStringErrorType::UnterminatedTripleQuotedString,
56+
)),
57+
..
58+
})) => ShellExecResult::ContinueLine,
4459
Err(err) => {
60+
// Check if the error is from an unclosed triple quoted string (which should always
61+
// continue)
62+
if let CompileError::Parse(ParseError {
63+
error: ParseErrorType::Lexical(LexicalErrorType::UnclosedStringError),
64+
raw_location,
65+
..
66+
}) = err
67+
{
68+
let loc = raw_location.start().to_usize();
69+
let mut iter = source.chars();
70+
if let Some(quote) = iter.nth(loc) {
71+
if iter.next() == Some(quote) && iter.next() == Some(quote) {
72+
return ShellExecResult::ContinueLine;
73+
}
74+
}
75+
};
76+
4577
// bad_error == true if we are handling an error that should be thrown even if we are continuing
4678
// if its an indentation error, set to true if we are continuing and the error is on column 0,
4779
// since indentations errors on columns other than 0 should be ignored.
@@ -50,10 +82,12 @@ fn shell_exec(
5082
let bad_error = match err {
5183
CompileError::Parse(ref p) => {
5284
match &p.error {
53-
ParseErrorType::Lexical(LexicalErrorType::IndentationError) => continuing, // && p.location.is_some()
85+
ParseErrorType::Lexical(LexicalErrorType::IndentationError) => {
86+
continuing_block
87+
} // && p.location.is_some()
5488
ParseErrorType::OtherError(msg) => {
5589
if msg.starts_with("Expected an indented block") {
56-
continuing
90+
continuing_block
5791
} else {
5892
true
5993
}
@@ -68,7 +102,7 @@ fn shell_exec(
68102
if empty_line_given || bad_error {
69103
ShellExecResult::PyErr(vm.new_syntax_error(&err, Some(source)))
70104
} else {
71-
ShellExecResult::Continue
105+
ShellExecResult::ContinueBlock
72106
}
73107
}
74108
}
@@ -93,10 +127,19 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> {
93127
println!("No previous history.");
94128
}
95129

96-
let mut continuing = false;
130+
// We might either be waiting to know if a block is complete, or waiting to know if a multiline
131+
// statement is complete. In the former case, we need to ensure that we read one extra new line
132+
// to know that the block is complete. In the latter, we can execute as soon as the statement is
133+
// valid.
134+
let mut continuing_block = false;
135+
let mut continuing_line = false;
97136

98137
loop {
99-
let prompt_name = if continuing { "ps2" } else { "ps1" };
138+
let prompt_name = if continuing_block || continuing_line {
139+
"ps2"
140+
} else {
141+
"ps1"
142+
};
100143
let prompt = vm
101144
.sys_module
102145
.get_attr(prompt_name, vm)
@@ -105,6 +148,8 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> {
105148
Ok(ref s) => s.as_str(),
106149
Err(_) => "",
107150
};
151+
152+
continuing_line = false;
108153
let result = match repl.readline(prompt) {
109154
ReadlineResult::Line(line) => {
110155
debug!("You entered {:?}", line);
@@ -120,39 +165,44 @@ pub fn run_shell(vm: &VirtualMachine, scope: Scope) -> PyResult<()> {
120165
}
121166
full_input.push('\n');
122167

123-
match shell_exec(vm, &full_input, scope.clone(), empty_line_given, continuing) {
168+
match shell_exec(
169+
vm,
170+
&full_input,
171+
scope.clone(),
172+
empty_line_given,
173+
continuing_block,
174+
) {
124175
ShellExecResult::Ok => {
125-
if continuing {
176+
if continuing_block {
126177
if empty_line_given {
127-
// We should be exiting continue mode
128-
continuing = false;
178+
// We should exit continue mode since the block successfully executed
179+
continuing_block = false;
129180
full_input.clear();
130-
Ok(())
131-
} else {
132-
// We should stay in continue mode
133-
continuing = true;
134-
Ok(())
135181
}
136182
} else {
137183
// We aren't in continue mode so proceed normally
138-
continuing = false;
139184
full_input.clear();
140-
Ok(())
141185
}
186+
Ok(())
187+
}
188+
// Continue, but don't change the mode
189+
ShellExecResult::ContinueLine => {
190+
continuing_line = true;
191+
Ok(())
142192
}
143-
ShellExecResult::Continue => {
144-
continuing = true;
193+
ShellExecResult::ContinueBlock => {
194+
continuing_block = true;
145195
Ok(())
146196
}
147197
ShellExecResult::PyErr(err) => {
148-
continuing = false;
198+
continuing_block = false;
149199
full_input.clear();
150200
Err(err)
151201
}
152202
}
153203
}
154204
ReadlineResult::Interrupt => {
155-
continuing = false;
205+
continuing_block = false;
156206
full_input.clear();
157207
let keyboard_interrupt =
158208
vm.new_exception_empty(vm.ctx.exceptions.keyboard_interrupt.to_owned());

‎vm/src/compiler.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,3 +49,10 @@ impl crate::convert::ToPyException for (CompileError, Option<&str>) {
4949
vm.new_syntax_error(&self.0, self.1)
5050
}
5151
}
52+
53+
#[cfg(any(feature = "parser", feature = "codegen"))]
54+
impl crate::convert::ToPyException for (CompileError, Option<&str>, bool) {
55+
fn to_pyexception(&self, vm: &crate::VirtualMachine) -> crate::builtins::PyBaseExceptionRef {
56+
vm.new_syntax_error_maybe_incomplete(&self.0, self.1, self.2)
57+
}
58+
}

‎vm/src/exceptions.rs

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -495,6 +495,7 @@ pub struct ExceptionZoo {
495495
pub not_implemented_error: &'static Py<PyType>,
496496
pub recursion_error: &'static Py<PyType>,
497497
pub syntax_error: &'static Py<PyType>,
498+
pub incomplete_input_error: &'static Py<PyType>,
498499
pub indentation_error: &'static Py<PyType>,
499500
pub tab_error: &'static Py<PyType>,
500501
pub system_error: &'static Py<PyType>,
@@ -743,6 +744,7 @@ impl ExceptionZoo {
743744
let recursion_error = PyRecursionError::init_builtin_type();
744745

745746
let syntax_error = PySyntaxError::init_builtin_type();
747+
let incomplete_input_error = PyIncompleteInputError::init_builtin_type();
746748
let indentation_error = PyIndentationError::init_builtin_type();
747749
let tab_error = PyTabError::init_builtin_type();
748750

@@ -817,6 +819,7 @@ impl ExceptionZoo {
817819
not_implemented_error,
818820
recursion_error,
819821
syntax_error,
822+
incomplete_input_error,
820823
indentation_error,
821824
tab_error,
822825
system_error,
@@ -965,6 +968,7 @@ impl ExceptionZoo {
965968
"end_offset" => ctx.none(),
966969
"text" => ctx.none(),
967970
});
971+
extend_exception!(PyIncompleteInputError, ctx, excs.incomplete_input_error);
968972
extend_exception!(PyIndentationError, ctx, excs.indentation_error);
969973
extend_exception!(PyTabError, ctx, excs.tab_error);
970974

@@ -1623,6 +1627,28 @@ pub(super) mod types {
16231627
}
16241628
}
16251629

1630+
#[pyexception(
1631+
name = "_IncompleteInputError",
1632+
base = "PySyntaxError",
1633+
ctx = "incomplete_input_error"
1634+
)]
1635+
#[derive(Debug)]
1636+
pub struct PyIncompleteInputError {}
1637+
1638+
#[pyexception]
1639+
impl PyIncompleteInputError {
1640+
#[pyslot]
1641+
#[pymethod(name = "__init__")]
1642+
pub(crate) fn slot_init(
1643+
zelf: PyObjectRef,
1644+
_args: FuncArgs,
1645+
vm: &VirtualMachine,
1646+
) -> PyResult<()> {
1647+
zelf.set_attr("name", vm.ctx.new_str("SyntaxError"), vm)?;
1648+
Ok(())
1649+
}
1650+
}
1651+
16261652
#[pyexception(name, base = "PySyntaxError", ctx = "indentation_error", impl)]
16271653
#[derive(Debug)]
16281654
pub struct PyIndentationError {}

‎vm/src/stdlib/ast.rs

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,7 @@ pub(crate) fn parse(
245245
let top = parser::parse(source, mode.into())
246246
.map_err(|parse_error| ParseError {
247247
error: parse_error.error,
248+
raw_location: parse_error.location,
248249
location: text_range_to_source_range(&source_code, parse_error.location)
249250
.start
250251
.to_source_location(),
@@ -295,8 +296,8 @@ pub const PY_COMPILE_FLAG_AST_ONLY: i32 = 0x0400;
295296
// The following flags match the values from Include/cpython/compile.h
296297
// Caveat emptor: These flags are undocumented on purpose and depending
297298
// on their effect outside the standard library is **unsupported**.
298-
const PY_CF_DONT_IMPLY_DEDENT: i32 = 0x200;
299-
const PY_CF_ALLOW_INCOMPLETE_INPUT: i32 = 0x4000;
299+
pub const PY_CF_DONT_IMPLY_DEDENT: i32 = 0x200;
300+
pub const PY_CF_ALLOW_INCOMPLETE_INPUT: i32 = 0x4000;
300301

301302
// __future__ flags - sync with Lib/__future__.py
302303
// TODO: These flags aren't being used in rust code

‎vm/src/stdlib/builtins.rs

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,8 @@ mod builtins {
186186
return Err(vm.new_value_error("compile() unrecognized flags".to_owned()));
187187
}
188188

189+
let allow_incomplete = !(flags & ast::PY_CF_ALLOW_INCOMPLETE_INPUT).is_zero();
190+
189191
if (flags & ast::PY_COMPILE_FLAG_AST_ONLY).is_zero() {
190192
#[cfg(not(feature = "compiler"))]
191193
{
@@ -207,14 +209,17 @@ mod builtins {
207209
args.filename.to_string_lossy().into_owned(),
208210
opts,
209211
)
210-
.map_err(|err| (err, Some(source)).to_pyexception(vm))?;
212+
.map_err(|err| {
213+
(err, Some(source), allow_incomplete).to_pyexception(vm)
214+
})?;
211215
Ok(code.into())
212216
}
213217
} else {
214218
let mode = mode_str
215219
.parse::<parser::Mode>()
216220
.map_err(|err| vm.new_value_error(err.to_string()))?;
217-
ast::parse(vm, source, mode).map_err(|e| (e, Some(source)).to_pyexception(vm))
221+
ast::parse(vm, source, mode)
222+
.map_err(|e| (e, Some(source), allow_incomplete).to_pyexception(vm))
218223
}
219224
}
220225
}
@@ -1056,6 +1061,7 @@ pub fn init_module(vm: &VirtualMachine, module: &Py<PyModule>) {
10561061
"NotImplementedError" => ctx.exceptions.not_implemented_error.to_owned(),
10571062
"RecursionError" => ctx.exceptions.recursion_error.to_owned(),
10581063
"SyntaxError" => ctx.exceptions.syntax_error.to_owned(),
1064+
"_IncompleteInputError" => ctx.exceptions.incomplete_input_error.to_owned(),
10591065
"IndentationError" => ctx.exceptions.indentation_error.to_owned(),
10601066
"TabError" => ctx.exceptions.tab_error.to_owned(),
10611067
"SystemError" => ctx.exceptions.system_error.to_owned(),

0 commit comments

Comments
 (0)