Skip to content

Commit 1b57f08

Browse files
youknowoneyouknowone
committed
Add source_path to ParseError
1 parent d516e6f commit 1b57f08

File tree

10 files changed

+60
-52
lines changed

10 files changed

+60
-52
lines changed

‎benches/execution.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ pub fn benchmark_file_execution(
5151
pub fn benchmark_file_parsing(group: &mut BenchmarkGroup<WallTime>, name: &str, contents: &str) {
5252
group.throughput(Throughput::Bytes(contents.len() as u64));
5353
group.bench_function(BenchmarkId::new("rustpython", name), |b| {
54-
b.iter(|| parse_program(contents).unwrap())
54+
b.iter(|| parse_program(contents, name).unwrap())
5555
});
5656
group.bench_function(BenchmarkId::new("cpython", name), |b| {
5757
let gil = cpython::Python::acquire_gil();

‎compiler/codegen/src/compile.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2707,7 +2707,7 @@ mod tests {
27072707
"source_path".to_owned(),
27082708
"<module>".to_owned(),
27092709
);
2710-
let ast = parser::parse_program(source).unwrap();
2710+
let ast = parser::parse_program(source, "<test>").unwrap();
27112711
let symbol_scope = SymbolTable::scan_program(&ast).unwrap();
27122712
compiler.compile_program(&ast, symbol_scope).unwrap();
27132713
compiler.pop_code_object()

‎compiler/parser/src/error.rs

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ impl From<FStringError> for LalrpopError<Location, Tok, LexicalError> {
124124
pub struct ParseError {
125125
pub error: ParseErrorType,
126126
pub location: Location,
127+
pub source_path: String,
127128
}
128129

129130
#[derive(Debug, PartialEq)]
@@ -141,21 +142,28 @@ pub enum ParseErrorType {
141142
}
142143

143144
/// Convert `lalrpop_util::ParseError` to our internal type
144-
impl From<LalrpopError<Location, Tok, LexicalError>> for ParseError {
145-
fn from(err: LalrpopError<Location, Tok, LexicalError>) -> Self {
145+
impl ParseError {
146+
pub(crate) fn from_lalrpop(
147+
err: LalrpopError<Location, Tok, LexicalError>,
148+
source_path: &str,
149+
) -> Self {
150+
let source_path = source_path.to_owned();
146151
match err {
147152
// TODO: Are there cases where this isn't an EOF?
148153
LalrpopError::InvalidToken { location } => ParseError {
149154
error: ParseErrorType::Eof,
150155
location,
156+
source_path,
151157
},
152158
LalrpopError::ExtraToken { token } => ParseError {
153159
error: ParseErrorType::ExtraToken(token.1),
154160
location: token.0,
161+
source_path,
155162
},
156163
LalrpopError::User { error } => ParseError {
157164
error: ParseErrorType::Lexical(error.error),
158165
location: error.location,
166+
source_path,
159167
},
160168
LalrpopError::UnrecognizedToken { token, expected } => {
161169
// Hacky, but it's how CPython does it. See PyParser_AddToken,
@@ -164,11 +172,13 @@ impl From<LalrpopError<Location, Tok, LexicalError>> for ParseError {
164172
ParseError {
165173
error: ParseErrorType::UnrecognizedToken(token.1, expected),
166174
location: token.0,
175+
source_path,
167176
}
168177
}
169178
LalrpopError::UnrecognizedEOF { location, .. } => ParseError {
170179
error: ParseErrorType::Eof,
171180
location,
181+
source_path,
172182
},
173183
}
174184
}

‎compiler/parser/src/fstring.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,7 @@ impl<'a> FStringParser<'a> {
293293

294294
fn parse_fstring_expr(source: &str) -> Result<Expr, ParseError> {
295295
let fstring_body = format!("({})", source);
296-
parse_expression(&fstring_body)
296+
parse_expression(&fstring_body, "<fstring>")
297297
}
298298

299299
/// Parse an fstring from a string, located at a certain position in the sourcecode.

‎compiler/parser/src/lib.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
//! use rustpython_parser::{parser, ast};
1212
//!
1313
//! let python_source = "print('Hello world')";
14-
//! let python_ast = parser::parse_expression(python_source).unwrap();
14+
//! let python_ast = parser::parse_expression(python_source, "<embedded>").unwrap();
1515
//!
1616
//! ```
1717

‎compiler/parser/src/parser.rs

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ use crate::python;
2020
*/
2121

2222
/// Parse a full python program, containing usually multiple lines.
23-
pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
24-
parse(source, Mode::Module).map(|top| match top {
23+
pub fn parse_program(source: &str, source_path: &str) -> Result<ast::Suite, ParseError> {
24+
parse(source, Mode::Module, source_path).map(|top| match top {
2525
ast::Mod::Module { body, .. } => body,
2626
_ => unreachable!(),
2727
})
@@ -33,7 +33,7 @@ pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
3333
/// ```
3434
/// extern crate num_bigint;
3535
/// use rustpython_parser::{parser, ast};
36-
/// let expr = parser::parse_expression("1 + 2").unwrap();
36+
/// let expr = parser::parse_expression("1 + 2", "<embedded>").unwrap();
3737
///
3838
/// assert_eq!(
3939
/// expr,
@@ -63,22 +63,22 @@ pub fn parse_program(source: &str) -> Result<ast::Suite, ParseError> {
6363
/// );
6464
///
6565
/// ```
66-
pub fn parse_expression(source: &str) -> Result<ast::Expr, ParseError> {
67-
parse(source, Mode::Expression).map(|top| match top {
66+
pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseError> {
67+
parse(source, Mode::Expression, path).map(|top| match top {
6868
ast::Mod::Expression { body } => *body,
6969
_ => unreachable!(),
7070
})
7171
}
7272

7373
// Parse a given source code
74-
pub fn parse(source: &str, mode: Mode) -> Result<ast::Mod, ParseError> {
74+
pub fn parse(source: &str, mode: Mode, source_path: &str) -> Result<ast::Mod, ParseError> {
7575
let lxr = lexer::make_tokenizer(source);
7676
let marker_token = (Default::default(), mode.to_marker(), Default::default());
7777
let tokenizer = iter::once(Ok(marker_token)).chain(lxr);
7878

7979
python::TopParser::new()
8080
.parse(tokenizer)
81-
.map_err(ParseError::from)
81+
.map_err(|e| ParseError::from_lalrpop(e, source_path))
8282
}
8383

8484
#[cfg(test)]
@@ -87,64 +87,64 @@ mod tests {
8787

8888
#[test]
8989
fn test_parse_empty() {
90-
let parse_ast = parse_program("").unwrap();
90+
let parse_ast = parse_program("", "<test>").unwrap();
9191
insta::assert_debug_snapshot!(parse_ast);
9292
}
9393

9494
#[test]
9595
fn test_parse_string() {
9696
let source = String::from("'Hello world'");
97-
let parse_ast = parse_program(&source).unwrap();
97+
let parse_ast = parse_program(&source, "<test>").unwrap();
9898
insta::assert_debug_snapshot!(parse_ast);
9999
}
100100

101101
#[test]
102102
fn test_parse_f_string() {
103103
let source = String::from("f'Hello world'");
104-
let parse_ast = parse_program(&source).unwrap();
104+
let parse_ast = parse_program(&source, "<test>").unwrap();
105105
insta::assert_debug_snapshot!(parse_ast);
106106
}
107107

108108
#[test]
109109
fn test_parse_print_hello() {
110110
let source = String::from("print('Hello world')");
111-
let parse_ast = parse_program(&source).unwrap();
111+
let parse_ast = parse_program(&source, "<test>").unwrap();
112112
insta::assert_debug_snapshot!(parse_ast);
113113
}
114114

115115
#[test]
116116
fn test_parse_print_2() {
117117
let source = String::from("print('Hello world', 2)");
118-
let parse_ast = parse_program(&source).unwrap();
118+
let parse_ast = parse_program(&source, "<test>").unwrap();
119119
insta::assert_debug_snapshot!(parse_ast);
120120
}
121121

122122
#[test]
123123
fn test_parse_kwargs() {
124124
let source = String::from("my_func('positional', keyword=2)");
125-
let parse_ast = parse_program(&source).unwrap();
125+
let parse_ast = parse_program(&source, "<test>").unwrap();
126126
insta::assert_debug_snapshot!(parse_ast);
127127
}
128128

129129
#[test]
130130
fn test_parse_if_elif_else() {
131131
let source = String::from("if 1: 10\nelif 2: 20\nelse: 30");
132-
let parse_ast = parse_program(&source).unwrap();
132+
let parse_ast = parse_program(&source, "<test>").unwrap();
133133
insta::assert_debug_snapshot!(parse_ast);
134134
}
135135

136136
#[test]
137137
fn test_parse_lambda() {
138138
let source = "lambda x, y: x * y"; // lambda(x, y): x * y";
139-
let parse_ast = parse_program(source).unwrap();
139+
let parse_ast = parse_program(source, "<test>").unwrap();
140140
insta::assert_debug_snapshot!(parse_ast);
141141
}
142142

143143
#[test]
144144
fn test_parse_tuples() {
145145
let source = "a, b = 4, 5";
146146

147-
insta::assert_debug_snapshot!(parse_program(source).unwrap());
147+
insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
148148
}
149149

150150
#[test]
@@ -155,48 +155,48 @@ class Foo(A, B):
155155
pass
156156
def method_with_default(self, arg='default'):
157157
pass";
158-
insta::assert_debug_snapshot!(parse_program(source).unwrap());
158+
insta::assert_debug_snapshot!(parse_program(source, "<test>").unwrap());
159159
}
160160

161161
#[test]
162162
fn test_parse_dict_comprehension() {
163163
let source = String::from("{x1: x2 for y in z}");
164-
let parse_ast = parse_expression(&source).unwrap();
164+
let parse_ast = parse_expression(&source, "<test>").unwrap();
165165
insta::assert_debug_snapshot!(parse_ast);
166166
}
167167

168168
#[test]
169169
fn test_parse_list_comprehension() {
170170
let source = String::from("[x for y in z]");
171-
let parse_ast = parse_expression(&source).unwrap();
171+
let parse_ast = parse_expression(&source, "<test>").unwrap();
172172
insta::assert_debug_snapshot!(parse_ast);
173173
}
174174

175175
#[test]
176176
fn test_parse_double_list_comprehension() {
177177
let source = String::from("[x for y, y2 in z for a in b if a < 5 if a > 10]");
178-
let parse_ast = parse_expression(&source).unwrap();
178+
let parse_ast = parse_expression(&source, "<test>").unwrap();
179179
insta::assert_debug_snapshot!(parse_ast);
180180
}
181181

182182
#[test]
183183
fn test_parse_generator_comprehension() {
184184
let source = String::from("(x for y in z)");
185-
let parse_ast = parse_expression(&source).unwrap();
185+
let parse_ast = parse_expression(&source, "<test>").unwrap();
186186
insta::assert_debug_snapshot!(parse_ast);
187187
}
188188

189189
#[test]
190190
fn test_parse_named_expression_generator_comprehension() {
191191
let source = String::from("(x := y + 1 for y in z)");
192-
let parse_ast = parse_expression(&source).unwrap();
192+
let parse_ast = parse_expression(&source, "<test>").unwrap();
193193
insta::assert_debug_snapshot!(parse_ast);
194194
}
195195

196196
#[test]
197197
fn test_parse_if_else_generator_comprehension() {
198198
let source = String::from("(x if y else y for y in z)");
199-
let parse_ast = parse_expression(&source).unwrap();
199+
let parse_ast = parse_expression(&source, "<test>").unwrap();
200200
insta::assert_debug_snapshot!(parse_ast);
201201
}
202202
}

‎compiler/parser/src/string.rs

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -88,56 +88,56 @@ mod tests {
8888
#[test]
8989
fn test_parse_string_concat() {
9090
let source = String::from("'Hello ' 'world'");
91-
let parse_ast = parse_program(&source).unwrap();
91+
let parse_ast = parse_program(&source, "<test>").unwrap();
9292
insta::assert_debug_snapshot!(parse_ast);
9393
}
9494

9595
#[test]
9696
fn test_parse_u_string_concat_1() {
9797
let source = String::from("'Hello ' u'world'");
98-
let parse_ast = parse_program(&source).unwrap();
98+
let parse_ast = parse_program(&source, "<test>").unwrap();
9999
insta::assert_debug_snapshot!(parse_ast);
100100
}
101101

102102
#[test]
103103
fn test_parse_u_string_concat_2() {
104104
let source = String::from("u'Hello ' 'world'");
105-
let parse_ast = parse_program(&source).unwrap();
105+
let parse_ast = parse_program(&source, "<test>").unwrap();
106106
insta::assert_debug_snapshot!(parse_ast);
107107
}
108108

109109
#[test]
110110
fn test_parse_f_string_concat_1() {
111111
let source = String::from("'Hello ' f'world'");
112-
let parse_ast = parse_program(&source).unwrap();
112+
let parse_ast = parse_program(&source, "<test>").unwrap();
113113
insta::assert_debug_snapshot!(parse_ast);
114114
}
115115

116116
#[test]
117117
fn test_parse_f_string_concat_2() {
118118
let source = String::from("'Hello ' f'world'");
119-
let parse_ast = parse_program(&source).unwrap();
119+
let parse_ast = parse_program(&source, "<test>").unwrap();
120120
insta::assert_debug_snapshot!(parse_ast);
121121
}
122122

123123
#[test]
124124
fn test_parse_f_string_concat_3() {
125125
let source = String::from("'Hello ' f'world{\"!\"}'");
126-
let parse_ast = parse_program(&source).unwrap();
126+
let parse_ast = parse_program(&source, "<test>").unwrap();
127127
insta::assert_debug_snapshot!(parse_ast);
128128
}
129129

130130
#[test]
131131
fn test_parse_u_f_string_concat_1() {
132132
let source = String::from("u'Hello ' f'world'");
133-
let parse_ast = parse_program(&source).unwrap();
133+
let parse_ast = parse_program(&source, "<test>").unwrap();
134134
insta::assert_debug_snapshot!(parse_ast);
135135
}
136136

137137
#[test]
138138
fn test_parse_u_f_string_concat_2() {
139139
let source = String::from("u'Hello ' f'world' '!'");
140-
let parse_ast = parse_program(&source).unwrap();
140+
let parse_ast = parse_program(&source, "<test>").unwrap();
141141
insta::assert_debug_snapshot!(parse_ast);
142142
}
143143
}

‎compiler/src/lib.rs

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -43,22 +43,18 @@ impl fmt::Display for CompileError {
4343

4444
impl CompileError {
4545
fn from_codegen(error: rustpython_codegen::error::CodegenError, source: &str) -> Self {
46-
CompileError {
46+
Self {
4747
error: error.error.into(),
4848
location: error.location,
4949
source_path: error.source_path,
5050
statement: get_statement(source, error.location),
5151
}
5252
}
53-
fn from_parse(
54-
error: rustpython_parser::error::ParseError,
55-
source: &str,
56-
source_path: String,
57-
) -> Self {
58-
CompileError {
53+
fn from_parse(error: rustpython_parser::error::ParseError, source: &str) -> Self {
54+
Self {
5955
error: error.error.into(),
6056
location: error.location,
61-
source_path,
57+
source_path: error.source_path,
6258
statement: get_statement(source, error.location),
6359
}
6460
}
@@ -83,9 +79,9 @@ pub fn compile(
8379
compile::Mode::Eval => parser::Mode::Expression,
8480
compile::Mode::Single | compile::Mode::BlockExpr => parser::Mode::Interactive,
8581
};
86-
let mut ast = match parser::parse(source, parser_mode) {
82+
let mut ast = match parser::parse(source, parser_mode, &source_path) {
8783
Ok(x) => x,
88-
Err(e) => return Err(CompileError::from_parse(e, source, source_path)),
84+
Err(e) => return Err(CompileError::from_parse(e, source)),
8985
};
9086
if opts.optimize > 0 {
9187
ast = ConstantOptimizer::new()
@@ -101,14 +97,14 @@ pub fn compile_symtable(
10197
mode: compile::Mode,
10298
source_path: &str,
10399
) -> Result<symboltable::SymbolTable, CompileError> {
104-
let parse_err = |e| CompileError::from_parse(e, source, source_path.to_owned());
100+
let parse_err = |e| CompileError::from_parse(e, source);
105101
let res = match mode {
106102
compile::Mode::Exec | compile::Mode::Single | compile::Mode::BlockExpr => {
107-
let ast = parser::parse_program(source).map_err(parse_err)?;
103+
let ast = parser::parse_program(source, source_path).map_err(parse_err)?;
108104
symboltable::SymbolTable::scan_program(&ast)
109105
}
110106
compile::Mode::Eval => {
111-
let expr = parser::parse_expression(source).map_err(parse_err)?;
107+
let expr = parser::parse_expression(source, source_path).map_err(parse_err)?;
112108
symboltable::SymbolTable::scan_expr(&expr)
113109
}
114110
};

‎examples/parse_folder.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,8 @@ fn parse_python_file(filename: &Path) -> ParsedFile {
8484
},
8585
Ok(source) => {
8686
let num_lines = source.to_string().lines().count();
87-
let result = parser::parse_program(&source).map_err(|e| e.to_string());
87+
let result = parser::parse_program(&source, &filename.to_string_lossy())
88+
.map_err(|e| e.to_string());
8889
ParsedFile {
8990
// filename: Box::new(filename.to_path_buf()),
9091
// code: source.to_string(),

0 commit comments

Comments
 (0)