Skip to content

Commit b112e46

Browse files
progvalprogval
committed
Fail early (ie. don't backtrack) after a keyword or an indentation error.
1 parent 52957a1 commit b112e46

File tree

8 files changed

+231
-83
lines changed

8 files changed

+231
-83
lines changed

‎src/bytes.rs

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -96,16 +96,16 @@ named!(pub bytes<StrSpan, Vec<u8>>,
9696
is_raw: call!(|i, s:StrSpan| Ok((i, s.fragment.0.contains('r') || s.fragment.0.contains('R'))), prefix) >>
9797
content: switch!(call!(|i| Ok((i, is_raw))),
9898
false => alt!(
99-
delimited!(tag!("'''"), call!(longbytes, '\''), tag!("'''"))
100-
| delimited!(tag!("\"\"\""), call!(longbytes, '"'), tag!("\"\"\""))
101-
| delimited!(char!('\''), call!(shortbytes, '\''), char!('\''))
102-
| delimited!(char!('"'), call!(shortbytes, '"'), char!('"'))
99+
delimited!(tag!("'''"), return_error!(call!(longbytes, '\'')), tag!("'''"))
100+
| delimited!(tag!("\"\"\""), return_error!(call!(longbytes, '"')), tag!("\"\"\""))
101+
| delimited!(char!('\''), return_error!(call!(shortbytes, '\'')), char!('\''))
102+
| delimited!(char!('"'), return_error!(call!(shortbytes, '"')), char!('"'))
103103
)
104104
| true => alt!(
105-
delimited!(tag!("'''"), call!(longrawbytes, '\''), tag!("'''"))
106-
| delimited!(tag!("\"\"\""), call!(longrawbytes, '"'), tag!("\"\"\""))
107-
| delimited!(char!('\''), call!(shortrawbytes, '\''), char!('\''))
108-
| delimited!(char!('"'), call!(shortrawbytes, '"'), char!('"'))
105+
delimited!(tag!("'''"), return_error!(call!(longrawbytes, '\'')), tag!("'''"))
106+
| delimited!(tag!("\"\"\""), return_error!(call!(longrawbytes, '"')), tag!("\"\"\""))
107+
| delimited!(char!('\''), return_error!(call!(shortrawbytes, '\'')), char!('\''))
108+
| delimited!(char!('"'), return_error!(call!(shortrawbytes, '"')), char!('"'))
109109
)
110110
) >> (content)
111111
)

‎src/errors.rs

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
2+
#[repr(u32)]
3+
pub enum PyParseError {
4+
UnexpectedIndent,
5+
ExpectedIndent,
6+
}
7+
impl From<PyParseError> for u32 {
8+
fn from(e: PyParseError) -> u32 {
9+
e as u32
10+
}
11+
}
12+
13+
#[cfg(test)]
14+
mod tests {
15+
use super::*;
16+
use nom;
17+
use nom::{Context, ErrorKind};
18+
use nom::types::CompleteStr;
19+
use nom_locate::LocatedSpan;
20+
21+
use helpers::*;
22+
use statements::statement;
23+
24+
25+
#[test]
26+
fn if_no_condition() {
27+
assert_eq!(statement(make_strspan("if:\n foo"), 0), Err(
28+
nom::Err::Failure(
29+
Context::Code(
30+
LocatedSpan { offset: 2, line: 1, fragment: CompleteStr(":\n foo") },
31+
ErrorKind::Alt
32+
)
33+
)
34+
));
35+
}
36+
}

‎src/expressions.rs

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,9 @@ named!(pub test<StrSpan, Box<Expression>>,
3232
left: call!(Self::or_test) >>
3333
right: opt!(do_parse!(
3434
ws_auto!(keyword!("if")) >>
35-
cond: call!(Self::or_test) >>
35+
cond: return_error!(call!(Self::or_test)) >>
3636
ws_auto!(keyword!("else")) >>
37-
right: call!(Self::test) >> (
37+
right: return_error!(call!(Self::test)) >> (
3838
(cond, right)
3939
)
4040
)) >> (
@@ -57,28 +57,26 @@ named!(test_nocond<StrSpan, Box<Expression>>,
5757

5858
// lambdef: 'lambda' [varargslist] ':' test
5959
named!(lambdef<StrSpan, Box<Expression>>,
60-
ws_auto!(do_parse!(
61-
keyword!("lambda") >>
60+
ws_auto!(preceded!(keyword!("lambda"), return_error!(do_parse!(
6261
args: opt!(varargslist) >>
6362
spaces!() >>
6463
char!(':') >>
6564
spaces!() >>
6665
code: call!(Self::test) >> (
6766
Box::new(Expression::Lambdef(args.unwrap_or_default(), code))
6867
)
69-
))
68+
))))
7069
);
7170

7271
// lambdef_nocond: 'lambda' [varargslist] ':' test_nocond
7372
named!(lambdef_nocond<StrSpan, Box<Expression>>,
74-
do_parse!(
75-
keyword!("lambda") >>
73+
ws_auto!(preceded!(keyword!("lambda"), return_error!(do_parse!(
7674
args: opt!(varargslist) >>
7775
char!(':') >>
7876
code: call!(Self::test_nocond) >> (
7977
Box::new(Expression::Lambdef(args.unwrap_or_default(), code))
8078
)
81-
)
79+
))))
8280
);
8381

8482
} // End ExpressionParser
@@ -426,7 +424,7 @@ named_args!(dictmaker(item1: DictItem) <StrSpan, Box<Expression>>,
426424
v.insert(0, item1.clone()); // FIXME: do not clone
427425
Box::new(Expression::DictLiteral(v))
428426
}}
429-
| preceded!(peek!(keyword!("for")), call!(Self::comp_for)) => { |comp| {
427+
| preceded!(peek!(keyword!("for")), return_error!(call!(Self::comp_for))) => { |comp| {
430428
Box::new(Expression::DictComp(Box::new(item1.clone()), comp)) // FIXME: do not clone
431429
}}
432430
)),
@@ -529,7 +527,7 @@ named_args!(comp_iter(acc: Vec<ComprehensionChunk>) <StrSpan, Vec<ComprehensionC
529527
);
530528

531529
named_args!(opt_comp_iter(acc: Vec<ComprehensionChunk>) <StrSpan, Vec<ComprehensionChunk>>,
532-
map!(opt!(call!(Self::comp_iter, acc.clone())), |r| r.unwrap_or(acc)) // FIXME: do not clone
530+
return_error!(map!(opt!(call!(Self::comp_iter, acc.clone())), |r| r.unwrap_or(acc))) // FIXME: do not clone
533531
);
534532

535533
// comp_for: [ASYNC] 'for' exprlist 'in' or_test [comp_iter]
@@ -541,11 +539,11 @@ named_args!(comp_for2(acc: Vec<ComprehensionChunk>) <StrSpan, Vec<ComprehensionC
541539
async: map!(opt!(terminated!(tag!("async"), space_sep!())), |o| o.is_some()) >>
542540
keyword!("for") >>
543541
spaces!() >>
544-
item: call!(Self::exprlist) >>
542+
item: return_error!(call!(Self::exprlist)) >>
545543
spaces!() >>
546544
keyword!("in") >>
547545
spaces!() >>
548-
iterator: map!(call!(Self::or_test), |e| *e) >>
546+
iterator: return_error!(map!(call!(Self::or_test), |e| *e)) >>
549547
spaces!() >>
550548
r: call!(Self::opt_comp_iter, { let mut acc = acc; acc.push(ComprehensionChunk::For { async, item, iterator }); acc }) >> (
551549
r
@@ -558,7 +556,7 @@ named_args!(comp_if(acc: Vec<ComprehensionChunk>) <StrSpan, Vec<ComprehensionChu
558556
do_parse!(
559557
keyword!("if") >>
560558
spaces!() >>
561-
cond: map!(call!(Self::test_nocond), |e| *e) >>
559+
cond: return_error!(map!(call!(Self::test_nocond), |e| *e)) >>
562560
spaces!() >>
563561
r: call!(Self::opt_comp_iter, { let mut acc = acc; acc.push(ComprehensionChunk::If { cond }); acc }) >> (
564562
r

‎src/functions.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ use ast::*;
1414
// decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
1515
named_args!(decorator(indent: usize) <StrSpan, Decorator>,
1616
do_parse!(
17-
count!(char!(' '), indent) >>
17+
indent!(indent) >>
1818
char!('@') >>
1919
name: ws_nonl!(call!(ImportParser::<NewlinesAreNotSpaces>::dotted_name)) >>
2020
args: opt!(ws_nonl!(delimited!(char!('('), ws_comm!(call!(ExpressionParser::<NewlinesAreSpaces>::arglist)), char!(')')))) >>
@@ -49,7 +49,7 @@ named_args!(pub decorated(indent: usize) <StrSpan, CompoundStatement>,
4949
// funcdef: 'def' NAME parameters ['->' test] ':' suite
5050
named_args!(funcdef(indent: usize, decorators: Vec<Decorator>) <StrSpan, CompoundStatement>,
5151
do_parse!(
52-
count!(char!(' '), indent) >>
52+
indent!(indent) >>
5353
async: opt!(tuple!(tag!("async"), space_sep_nonl)) >>
5454
tag!("def") >>
5555
space_sep_nonl >>
@@ -68,7 +68,7 @@ named_args!(funcdef(indent: usize, decorators: Vec<Decorator>) <StrSpan, Compoun
6868
// classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
6969
named_args!(classdef(indent: usize, decorators: Vec<Decorator>) <StrSpan, CompoundStatement>,
7070
do_parse!(
71-
count!(char!(' '), indent) >>
71+
indent!(indent) >>
7272
tag!("class") >>
7373
space_sep_nonl >>
7474
name: name >>

‎src/helpers.rs

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,3 +205,82 @@ pub(crate) fn first_word(i: StrSpan) -> Result<(StrSpan, &str), ::nom::Err<StrSp
205205
Err(e) => Err(e),
206206
}
207207
}
208+
209+
// https://github.com/Geal/nom/pull/800
210+
macro_rules! fold_many1_fixed(
211+
($i:expr, $submac:ident!( $($args:tt)* ), $init:expr, $f:expr) => (
212+
{
213+
use nom;
214+
use nom::lib::std::result::Result::*;
215+
use nom::{Err,Needed,InputLength,Context,AtEof};
216+
217+
match $submac!($i, $($args)*) {
218+
Err(Err::Error(_)) => Err(Err::Error(
219+
error_position!($i, nom::ErrorKind::Many1)
220+
)),
221+
Err(Err::Failure(_)) => Err(Err::Failure(
222+
error_position!($i, nom::ErrorKind::Many1)
223+
)),
224+
Err(Err::Incomplete(i)) => Err(Err::Incomplete(i)),
225+
Ok((i1,o1)) => {
226+
let f = $f;
227+
let mut acc = f($init, o1);
228+
let mut input = i1;
229+
let mut incomplete: nom::lib::std::option::Option<Needed> =
230+
nom::lib::std::option::Option::None;
231+
let mut failure: nom::lib::std::option::Option<Context<_,_>> =
232+
nom::lib::std::option::Option::None;
233+
loop {
234+
match $submac!(input, $($args)*) {
235+
Err(Err::Error(_)) => {
236+
break;
237+
},
238+
Err(Err::Incomplete(i)) => {
239+
incomplete = nom::lib::std::option::Option::Some(i);
240+
break;
241+
},
242+
Err(Err::Failure(e)) => {
243+
failure = nom::lib::std::option::Option::Some(e);
244+
break;
245+
},
246+
Ok((i, o)) => {
247+
if i.input_len() == input.input_len() {
248+
if !i.at_eof() {
249+
failure = nom::lib::std::option::Option::Some(error_position!(i, nom::ErrorKind::Many1));
250+
}
251+
break;
252+
}
253+
acc = f(acc, o);
254+
input = i;
255+
}
256+
}
257+
}
258+
259+
match failure {
260+
nom::lib::std::option::Option::Some(e) => Err(Err::Failure(e)),
261+
nom::lib::std::option::Option::None => match incomplete {
262+
nom::lib::std::option::Option::Some(i) => nom::need_more($i, i),
263+
nom::lib::std::option::Option::None => Ok((input, acc))
264+
}
265+
}
266+
}
267+
}
268+
}
269+
);
270+
($i:expr, $f:expr, $init:expr, $fold_f:expr) => (
271+
fold_many_fixed1!($i, call!($f), $init, $fold_f);
272+
);
273+
);
274+
275+
macro_rules! indent {
276+
($i:expr, $nb_spaces:expr) => {{
277+
use nom::ErrorKind;
278+
use $crate::errors::PyParseError;
279+
count!($i, char!(' '), $nb_spaces).and_then(|(i2,_)|
280+
return_error!(i2,
281+
ErrorKind::Custom(PyParseError::UnexpectedIndent.into()),
282+
not!(peek!(char!(' ')))
283+
)
284+
)
285+
}}
286+
}

‎src/lib.rs

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ mod bytes;
9191
mod numbers;
9292
pub mod ast;
9393
pub mod visitors;
94+
pub mod errors;
9495

9596
use helpers::*;
9697
use statements::*;
@@ -113,8 +114,9 @@ named_attr!(#[doc = "Parses a module or sequence of commands."],
113114
pub file_input <StrSpan, Vec<Statement>>,
114115
fold_many0!(
115116
alt!(
116-
call!(statement, 0) => { |s| Some(s) }
117-
| newline => { |_| None }
117+
newline => { |_| None }
118+
| eof!() => { |_| None }
119+
| call!(statement, 0) => { |s| Some(s) }
118120
),
119121
Vec::new(),
120122
|acc: Vec<_>, item| { let mut acc = acc; if let Some(s) = item { acc.extend(s); } acc }

0 commit comments

Comments
 (0)