Skip to content

Commit 43b0760

Browse files
coolreader18coolreader18
committed
Merge branch 'master' into coolreader18/py_class-proc-macro
2 parents 51b7660 + 42768b2 commit 43b0760

File tree

12 files changed

+238
-181
lines changed

12 files changed

+238
-181
lines changed

‎parser/.gitignore

Lines changed: 0 additions & 4 deletions
This file was deleted.

‎parser/build.rs

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,5 @@
11
use lalrpop;
22

33
fn main() {
4-
lalrpop::Configuration::new()
5-
.generate_in_source_tree()
6-
.process()
7-
.unwrap();
4+
lalrpop::process_root().unwrap()
85
}

‎parser/src/lexer.rs

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -466,8 +466,16 @@ where
466466
loop {
467467
match self.next_char() {
468468
Some('\\') => {
469-
if is_raw {
469+
if self.chr0 == Some(quote_char) {
470+
string_content.push(quote_char);
471+
self.next_char();
472+
} else if is_raw {
470473
string_content.push('\\');
474+
if let Some(c) = self.next_char() {
475+
string_content.push(c)
476+
} else {
477+
return Err(LexicalError::StringError);
478+
}
471479
} else {
472480
match self.next_char() {
473481
Some('\\') => {
@@ -711,7 +719,6 @@ where
711719
let tok_start = self.get_pos();
712720
self.next_char();
713721
let tok_end = self.get_pos();
714-
println!("Emoji: {}", c);
715722
return Some(Ok((
716723
tok_start,
717724
Tok::Name {
@@ -1438,7 +1445,7 @@ mod tests {
14381445

14391446
#[test]
14401447
fn test_string() {
1441-
let source = String::from(r#""double" 'single' 'can\'t' "\\\"" '\t\r\n' '\g'"#);
1448+
let source = String::from(r#""double" 'single' 'can\'t' "\\\"" '\t\r\n' '\g' r'raw\''"#);
14421449
let tokens = lex_source(&source);
14431450
assert_eq!(
14441451
tokens,
@@ -1467,6 +1474,10 @@ mod tests {
14671474
value: String::from("\\g"),
14681475
is_fstring: false,
14691476
},
1477+
Tok::String {
1478+
value: String::from("raw\'"),
1479+
is_fstring: false,
1480+
},
14701481
]
14711482
);
14721483
}

‎parser/src/lib.rs

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
#[macro_use]
22
extern crate log;
3+
use lalrpop_util::lalrpop_mod;
34

45
pub mod ast;
56
pub mod error;
67
mod fstring;
78
pub mod lexer;
89
pub mod parser;
9-
#[cfg_attr(rustfmt, rustfmt_skip)]
10-
mod python;
10+
lalrpop_mod!(
11+
#[allow(clippy::all)]
12+
python
13+
);
1114
pub mod token;

‎parser/src/python.lalrpop

Lines changed: 23 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
// See also: https://github.com/antlr/grammars-v4/blob/master/python3/Python3.g4
33
// See also: file:///usr/share/doc/python/html/reference/compound_stmts.html#function-definitions
44
// See also: https://greentreesnakes.readthedocs.io/en/latest/nodes.html#keyword
5-
#![allow(unknown_lints,clippy)]
65

76
use std::iter::FromIterator;
87

@@ -337,10 +336,7 @@ CompoundStatement: ast::LocatedStatement = {
337336
IfStatement: ast::LocatedStatement = {
338337
<loc:@L> "if" <t:Test> ":" <s1:Suite> <s2:(@L "elif" Test ":" Suite)*> <s3:("else" ":" Suite)?> => {
339338
// Determine last else:
340-
let mut last = match s3 {
341-
Some(s) => Some(s.2),
342-
None => None,
343-
};
339+
let mut last = s3.map(|s| s.2);
344340

345341
// handle elif:
346342
for i in s2.into_iter().rev() {
@@ -360,10 +356,7 @@ IfStatement: ast::LocatedStatement = {
360356

361357
WhileStatement: ast::LocatedStatement = {
362358
<loc:@L> "while" <e:Test> ":" <s:Suite> <s2:("else" ":" Suite)?> => {
363-
let or_else = match s2 {
364-
Some(s) => Some(s.2),
365-
None => None,
366-
};
359+
let or_else = s2.map(|s| s.2);
367360
ast::LocatedStatement {
368361
location: loc,
369362
node: ast::Statement::While { test: e, body: s, orelse: or_else },
@@ -373,10 +366,7 @@ WhileStatement: ast::LocatedStatement = {
373366

374367
ForStatement: ast::LocatedStatement = {
375368
<loc:@L> "for" <e:ExpressionList> "in" <t:TestList> ":" <s:Suite> <s2:("else" ":" Suite)?> => {
376-
let or_else = match s2 {
377-
Some(s) => Some(s.2),
378-
None => None,
379-
};
369+
let or_else = s2.map(|s| s.2);
380370
ast::LocatedStatement {
381371
location: loc,
382372
node: ast::Statement::For {
@@ -389,14 +379,8 @@ ForStatement: ast::LocatedStatement = {
389379

390380
TryStatement: ast::LocatedStatement = {
391381
<loc:@L> "try" ":" <body:Suite> <handlers:ExceptClause*> <else_suite:("else" ":" Suite)?> <finally:("finally" ":" Suite)?> => {
392-
let or_else = match else_suite {
393-
Some(s) => Some(s.2),
394-
None => None,
395-
};
396-
let finalbody = match finally {
397-
Some(s) => Some(s.2),
398-
None => None,
399-
};
382+
let or_else = else_suite.map(|s| s.2);
383+
let finalbody = finally.map(|s| s.2);
400384
ast::LocatedStatement {
401385
location: loc,
402386
node: ast::Statement::Try {
@@ -437,10 +421,7 @@ WithStatement: ast::LocatedStatement = {
437421

438422
WithItem: ast::WithItem = {
439423
<t:Test> <n:("as" Expression)?> => {
440-
let optional_vars = match n {
441-
Some(val) => Some(val.1),
442-
None => None,
443-
};
424+
let optional_vars = n.map(|val| val.1);
444425
ast::WithItem { context_expr: t, optional_vars }
445426
},
446427
};
@@ -461,27 +442,19 @@ FuncDef: ast::LocatedStatement = {
461442
};
462443

463444
Parameters: ast::Parameters = {
464-
"(" <a: (TypedArgsList<TypedParameter>)?> ")" => {
465-
match a {
466-
Some(a) => a,
467-
None => Default::default(),
468-
}
445+
"(" <a: (ParameterList<TypedParameter>)?> ")" => {
446+
a.unwrap_or_else(Default::default)
469447
},
470448
};
471449

472-
// parameters are (String, None), kwargs are (String, Some(Test)) where Test is
473-
// the default
474450
// Note that this is a macro which is used once for function defs, and
475451
// once for lambda defs.
476-
TypedArgsList<ArgType>: ast::Parameters = {
477-
<param1:TypedParameters<ArgType>> <args2:("," ParameterListStarArgs<ArgType>)?> => {
452+
ParameterList<ArgType>: ast::Parameters = {
453+
<param1:ParameterDefs<ArgType>> <args2:("," ParameterListStarArgs<ArgType>)?> ","? => {
478454
let (names, default_elements) = param1;
479455

480456
// Now gather rest of parameters:
481-
let (vararg, kwonlyargs, kw_defaults, kwarg) = match args2 {
482-
Some((_, x)) => x,
483-
None => (None, vec![], vec![], None),
484-
};
457+
let (vararg, kwonlyargs, kw_defaults, kwarg) = args2.map_or((None, vec![], vec![], None), |x| x.1);
485458

486459
ast::Parameters {
487460
args: names,
@@ -492,7 +465,7 @@ TypedArgsList<ArgType>: ast::Parameters = {
492465
kw_defaults: kw_defaults,
493466
}
494467
},
495-
<param1:TypedParameters<ArgType>> <kw:("," KwargParameter<ArgType>)> => {
468+
<param1:ParameterDefs<ArgType>> <kw:("," KwargParameter<ArgType>)> ","? => {
496469
let (names, default_elements) = param1;
497470

498471
// Now gather rest of parameters:
@@ -510,7 +483,7 @@ TypedArgsList<ArgType>: ast::Parameters = {
510483
kw_defaults: kw_defaults,
511484
}
512485
},
513-
<params:ParameterListStarArgs<ArgType>> => {
486+
<params:ParameterListStarArgs<ArgType>> ","? => {
514487
let (vararg, kwonlyargs, kw_defaults, kwarg) = params;
515488
ast::Parameters {
516489
args: vec![],
@@ -521,7 +494,7 @@ TypedArgsList<ArgType>: ast::Parameters = {
521494
kw_defaults: kw_defaults,
522495
}
523496
},
524-
<kw:KwargParameter<ArgType>> => {
497+
<kw:KwargParameter<ArgType>> ","? => {
525498
ast::Parameters {
526499
args: vec![],
527500
kwonlyargs: vec![],
@@ -535,8 +508,8 @@ TypedArgsList<ArgType>: ast::Parameters = {
535508

536509
// Use inline here to make sure the "," is not creating an ambiguity.
537510
#[inline]
538-
TypedParameters<ArgType>: (Vec<ast::Parameter>, Vec<ast::Expression>) = {
539-
<param1:TypedParameterDef<ArgType>> <param2:("," TypedParameterDef<ArgType>)*> => {
511+
ParameterDefs<ArgType>: (Vec<ast::Parameter>, Vec<ast::Expression>) = {
512+
<param1:ParameterDef<ArgType>> <param2:("," ParameterDef<ArgType>)*> => {
540513
// Combine first parameters:
541514
let mut args = vec![param1];
542515
args.extend(param2.into_iter().map(|x| x.1));
@@ -564,7 +537,7 @@ TypedParameters<ArgType>: (Vec<ast::Parameter>, Vec<ast::Expression>) = {
564537
}
565538
};
566539

567-
TypedParameterDef<ArgType>: (ast::Parameter, Option<ast::Expression>) = {
540+
ParameterDef<ArgType>: (ast::Parameter, Option<ast::Expression>) = {
568541
<i:ArgType> => (i, None),
569542
<i:ArgType> "=" <e:Test> => (i, Some(e)),
570543
};
@@ -580,8 +553,11 @@ TypedParameter: ast::Parameter = {
580553
},
581554
};
582555

556+
// Use inline here to make sure the "," is not creating an ambiguity.
557+
// TODO: figure out another grammar that makes this inline no longer required.
558+
#[inline]
583559
ParameterListStarArgs<ArgType>: (Option<Option<ast::Parameter>>, Vec<ast::Parameter>, Vec<Option<ast::Expression>>, Option<Option<ast::Parameter>>) = {
584-
"*" <va:ArgType?> <kw:("," TypedParameterDef<ArgType>)*> <kwarg:("," KwargParameter<ArgType>)?> => {
560+
"*" <va:ArgType?> <kw:("," ParameterDef<ArgType>)*> <kwarg:("," KwargParameter<ArgType>)?> => {
585561
// Extract keyword arguments:
586562
let mut kwonlyargs = vec![];
587563
let mut kw_defaults = vec![];
@@ -590,10 +566,7 @@ ParameterListStarArgs<ArgType>: (Option<Option<ast::Parameter>>, Vec<ast::Parame
590566
kw_defaults.push(value);
591567
}
592568

593-
let kwarg = match kwarg {
594-
Some((_, name)) => Some(name),
595-
None => None,
596-
};
569+
let kwarg = kwarg.map(|n| n.1);
597570

598571
(Some(va), kwonlyargs, kw_defaults, kwarg)
599572
}
@@ -684,7 +657,7 @@ Test: ast::Expression = {
684657
};
685658

686659
LambdaDef: ast::Expression = {
687-
"lambda" <p:TypedArgsList<UntypedParameter>?> ":" <b:Test> =>
660+
"lambda" <p:ParameterList<UntypedParameter>?> ":" <b:Test> =>
688661
ast::Expression::Lambda {
689662
args: p.unwrap_or(Default::default()),
690663
body:Box::new(b)

‎tests/snippets/bytes.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
assert b'foobar'.__eq__(2) == NotImplemented
2+
assert b'foobar'.__ne__(2) == NotImplemented
3+
assert b'foobar'.__gt__(2) == NotImplemented
4+
assert b'foobar'.__ge__(2) == NotImplemented
5+
assert b'foobar'.__lt__(2) == NotImplemented
6+
assert b'foobar'.__le__(2) == NotImplemented

‎tests/snippets/class.py

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,7 @@ def kungfu(x):
3737
assert x == 3
3838

3939

40-
# TODO:
41-
# assert Bar.__doc__ == " W00t "
40+
assert Bar.__doc__ == " W00t "
4241

4342
bar = Bar(42)
4443

@@ -117,3 +116,31 @@ def f(self):
117116
assert type(a) is super
118117
assert a.conjugate() == 1
119118

119+
120+
class T1:
121+
"test1"
122+
123+
assert T1.__doc__ == "test1"
124+
125+
class T2:
126+
'''test2'''
127+
128+
assert T2.__doc__ == "test2"
129+
130+
class T3:
131+
"""
132+
test3
133+
"""
134+
135+
assert T3.__doc__ == "\n test3\n "
136+
137+
class T4:
138+
139+
"""test4"""
140+
141+
def t1(self):
142+
"""t1"""
143+
pass
144+
145+
assert T4.__doc__ == "test4"
146+
assert T4.t1.__doc__ == "t1"

‎tests/snippets/function.py

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,39 @@
11
def foo():
2+
"""test"""
23
return 42
34

45
assert foo() == 42
6+
assert foo.__doc__ == "test"
7+
8+
def my_func(a,):
9+
return a+2
10+
11+
assert my_func(2) == 4
12+
13+
14+
def f1():
15+
16+
"""test1"""
17+
pass
18+
19+
assert f1.__doc__ == "test1"
20+
21+
def f2():
22+
'''test2'''
23+
pass
24+
25+
assert f2.__doc__ == "test2"
26+
27+
def f3():
28+
"""
29+
test3
30+
"""
31+
pass
32+
33+
assert f3.__doc__ == "\n test3\n "
34+
35+
def f4():
36+
"test4"
37+
pass
38+
39+
assert f4.__doc__ == "test4"

0 commit comments

Comments
 (0)