From b6167c80d65366606474c1e68cc565867bafc1e0 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Sat, 18 Jan 2025 00:08:35 +0100 Subject: [PATCH 01/21] fix: repair comprehension ifs ... again (#7) * fix: repair comprehension ifs ... again * make example more complex --- src/unparser.rs | 12 ++---------- test_files/simple_comprehensions.py | 6 ++++++ 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/src/unparser.rs b/src/unparser.rs index 02c97b2..652f48f 100644 --- a/src/unparser.rs +++ b/src/unparser.rs @@ -1152,16 +1152,8 @@ impl Unparser { self.write_str(" in "); self.unparse_expr(&node.iter); for if_ in &node.ifs { - self.write_str(" "); - match if_ { - Expr::Name(name) => { - self.write_str("if "); - self.write_str(&name.id); - } - _ => { - self.unparse_expr(if_); - } - } + self.write_str(" if "); + self.unparse_expr(if_); } } diff --git a/test_files/simple_comprehensions.py b/test_files/simple_comprehensions.py index 4da6019..2624a6f 100644 --- a/test_files/simple_comprehensions.py +++ b/test_files/simple_comprehensions.py @@ -8,6 +8,12 @@ with_outer_if = (value for value in random.choices(string.ascii_letters) if value) +with_outer_if_not = ( + value + for value in random.choices(string.ascii_letters) + if not value + if value != value +) with_inner_if_else = ( value if value else "missing!" for value in random.choices(string.ascii_letters) ) From 4631289338e95e33dd86f27451df624f44c91b42 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Sat, 18 Jan 2025 00:09:49 +0100 Subject: [PATCH 02/21] chore: bump version to 0.1.3 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8fa86e4..94fcc74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -483,7 +483,7 @@ dependencies = [ [[package]] name = "rustpython-unparser" -version = "0.1.2" +version = "0.1.3" dependencies = [ "pretty_assertions", "rand", diff --git a/Cargo.toml b/Cargo.toml index 610e535..1b9d9b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustpython-unparser" -version = "0.1.2" +version = "0.1.3" edition = "2021" authors = ["Jan Vollmer "] readme = "README.md" From 338c72450c2f23f02e93ae2455d3711b89567da2 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Sat, 18 Jan 2025 00:25:48 +0100 Subject: [PATCH 03/21] fix: precedence for starred expr (#8) --- src/unparser.rs | 5 +++-- test_files/starred.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 test_files/starred.py diff --git a/src/unparser.rs b/src/unparser.rs index 652f48f..21f7e6e 100644 --- a/src/unparser.rs +++ b/src/unparser.rs @@ -44,7 +44,6 @@ impl Precedence { } } -#[allow(dead_code)] const EXPR_PRECEDENCE: usize = 9; fn get_precedence(node: &Expr) -> usize { @@ -1080,7 +1079,9 @@ impl Unparser { } fn unparse_expr_starred(&mut self, node: &ExprStarred) { self.write_str("*"); - self.unparse_expr(&node.value) + self.with_precedence_num(EXPR_PRECEDENCE, |prec_self| { + prec_self.unparse_expr(&node.value); + }); } fn unparse_expr_name(&mut self, node: &ExprName) { diff --git a/test_files/starred.py b/test_files/starred.py new file mode 100644 index 0000000..fb6c769 --- /dev/null +++ b/test_files/starred.py @@ -0,0 +1,3 @@ +var1: list[str] = [] + +var = [*(var1 or ())] From 79ce7d9cc441300b378e887c750b12693db58f2b Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Sat, 18 Jan 2025 01:42:58 +0100 Subject: [PATCH 04/21] fix: cover more extreme edge cases for f-string (#9) --- Cargo.lock | 1 + Cargo.toml | 2 +- src/lib.rs | 2 +- src/unparser.rs | 44 ++++++++++++++++++++++++++++++----- src/utils.rs | 9 +++++++ test_files/f_string.py | 22 ++++++++++++++++++ test_files/simple_f_string.py | 8 ------- 7 files changed, 72 insertions(+), 16 deletions(-) create mode 100644 src/utils.rs create mode 100644 test_files/f_string.py delete mode 100644 test_files/simple_f_string.py diff --git a/Cargo.lock b/Cargo.lock index 94fcc74..f75921c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -419,6 +419,7 @@ checksum = "4cdaf8ee5c1473b993b398c174641d3aa9da847af36e8d5eb8291930b72f31a5" dependencies = [ "is-macro", "malachite-bigint", + "rustpython-literal", "rustpython-parser-core", "static_assertions", ] diff --git a/Cargo.toml b/Cargo.toml index 1b9d9b1..4094e51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,6 @@ rustpython-literal = ">=0.4.0" [dev-dependencies] rustpython-parser = "0.4.0" -rustpython-ast = { version = "0.4.0", features = ["fold"] } +rustpython-ast = { version = "0.4.0", features = ["fold", "unparse"] } rand = "0.8.5" pretty_assertions = "1.4.1" diff --git a/src/lib.rs b/src/lib.rs index 528e77a..85c9324 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,5 @@ pub mod unparser; - +mod utils; pub use crate::unparser::Unparser; #[cfg(test)] diff --git a/src/unparser.rs b/src/unparser.rs index 21f7e6e..01b0b58 100644 --- a/src/unparser.rs +++ b/src/unparser.rs @@ -16,6 +16,8 @@ use rustpython_ast::{ use rustpython_ast::{Constant, ConversionFlag, Int}; use std::ops::Deref; +use crate::utils::replace_first_and_last; + enum Precedence { NamedExpr = 1, Tuple = 2, @@ -986,6 +988,8 @@ impl Unparser { self.write_str("f"); } let mut expr_source = String::new(); + + let mut formatted_values_sources: Vec = Vec::new(); for expr in node.values.iter() { let mut inner_unparser = Unparser::new(); match expr { @@ -996,22 +1000,50 @@ impl Unparser { } else { unreachable!() } + expr_source += inner_unparser.source.as_str(); + } + Expr::FormattedValue(formatted) => { + expr_source += &("{".to_owned() + + formatted_values_sources.len().to_string().as_str() + + "}"); + inner_unparser.unparse_expr_formatted_value(formatted); + formatted_values_sources.push(inner_unparser.source); } _ => { inner_unparser.unparse_expr(expr); + expr_source += inner_unparser.source.as_str(); } } - - expr_source += inner_unparser.source.as_str(); } if is_spec { + for (i, formatted) in formatted_values_sources.iter().enumerate() { + let to_replace = "{".to_owned() + i.to_string().as_str() + "}"; + expr_source = expr_source.replace(&to_replace, formatted) + } self.write_str(&expr_source); } else { - let escaped_source = rustpython_literal::escape::UnicodeEscape::new_repr(&expr_source) - .str_repr() - .to_string() - .unwrap(); + let mut escaped_source = + rustpython_literal::escape::UnicodeEscape::new_repr(&expr_source) + .str_repr() + .to_string() + .unwrap(); + for (i, formatted) in formatted_values_sources.iter().enumerate() { + let to_replace = "{".to_owned() + i.to_string().as_str() + "}"; + escaped_source = escaped_source.replace(&to_replace, formatted) + } + + let has_single = escaped_source.contains("'"); + let has_double = escaped_source.contains("\""); + let has_single_doc = escaped_source.contains("'''"); + if has_single && has_double && has_single_doc { + escaped_source = replace_first_and_last(&escaped_source, "\"\"\"") + } else if has_single && has_double { + escaped_source = replace_first_and_last(&escaped_source, "'''") + } else if has_single { + escaped_source = replace_first_and_last(&escaped_source, "\"") + } + self.write_str(&escaped_source); } } diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..95280ae --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,9 @@ +pub fn replace_first_and_last(s: &str, replacement: &str) -> String { + if s.len() <= 1 { + return replacement.to_string() + replacement; + } + + let middle = &s[1..s.len() - 1]; + + format!("{}{}{}", replacement, middle, replacement) +} diff --git a/test_files/f_string.py b/test_files/f_string.py new file mode 100644 index 0000000..8a4bb93 --- /dev/null +++ b/test_files/f_string.py @@ -0,0 +1,22 @@ +world = "World" +empty_f_string = f"" # noqa: F541 +f"{world!a:}None{empty_f_string!s:}None" +answer = 42.000001 +f"{answer:.03f}" +f"'\"'''\"\"\"{{}}\\" # noqa +if __name__ == "__main__": + print(f"Hello {world}!") + +# fmt: off +lines = "\n".join( + f'''{make_tag({"rect"}, x=0, y=offset, width=char_width * width, height=line_height + 0.25)}''' # noqa + for line_no, offset in enumerate(line_offsets) # noqa + ) + +tag_attribs = " ".join( + ( + f'''{k.lstrip('_').replace('_', '-')}="{stringify(v)}"''' # noqa + for (k, v) in attribs.items() # noqa + ) +) +# fmt: on diff --git a/test_files/simple_f_string.py b/test_files/simple_f_string.py deleted file mode 100644 index 2599915..0000000 --- a/test_files/simple_f_string.py +++ /dev/null @@ -1,8 +0,0 @@ -world = "World" -empty_f_string = f"" # noqa: F541 -f"{world!a:}None{empty_f_string!s:}None" -answer = 42.000001 -f"{answer:.03f}" -f"'\"'''\"\"\"{{}}\\" # noqa -if __name__ == "__main__": - print(f"Hello {world}!") From bec6cbed87731e884edaa2f57499a7f0ebcd6710 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Sat, 18 Jan 2025 01:55:51 +0100 Subject: [PATCH 05/21] fix: improve f-strings even more (#10) * fix: improve f-strings even more * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- src/unparser.rs | 14 +++++++++++--- test_files/f_string.py | 3 +++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/unparser.rs b/src/unparser.rs index 01b0b58..8b6368f 100644 --- a/src/unparser.rs +++ b/src/unparser.rs @@ -1035,10 +1035,18 @@ impl Unparser { let has_single = escaped_source.contains("'"); let has_double = escaped_source.contains("\""); - let has_single_doc = escaped_source.contains("'''"); - if has_single && has_double && has_single_doc { + + if has_single + && has_double + && escaped_source.starts_with("\"") + && escaped_source.ends_with("\"") + { escaped_source = replace_first_and_last(&escaped_source, "\"\"\"") - } else if has_single && has_double { + } else if has_single + && has_double + && escaped_source.starts_with("'") + && escaped_source.ends_with("'") + { escaped_source = replace_first_and_last(&escaped_source, "'''") } else if has_single { escaped_source = replace_first_and_last(&escaped_source, "\"") diff --git a/test_files/f_string.py b/test_files/f_string.py index 8a4bb93..9bf4ea8 100644 --- a/test_files/f_string.py +++ b/test_files/f_string.py @@ -19,4 +19,7 @@ for (k, v) in attribs.items() # noqa ) ) + +completion_init_lines = [f"source '{completion_path}'"] # noqa + # fmt: on From 9355635add5af2ec680077f35abef2e5d9e7e2f8 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Sat, 18 Jan 2025 01:59:42 +0100 Subject: [PATCH 06/21] chore: bump version to 0.1.4 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f75921c..a323c4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -484,7 +484,7 @@ dependencies = [ [[package]] name = "rustpython-unparser" -version = "0.1.3" +version = "0.1.4" dependencies = [ "pretty_assertions", "rand", diff --git a/Cargo.toml b/Cargo.toml index 4094e51..fad0466 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustpython-unparser" -version = "0.1.3" +version = "0.1.4" edition = "2021" authors = ["Jan Vollmer "] readme = "README.md" From 8a36fad56e33069a168cceb57f5845a9b80ff829 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Fri, 24 Jan 2025 15:05:56 +0100 Subject: [PATCH 07/21] feat: bring flay's transformer to rustpython-unparser --- Cargo.toml | 3 + README.md | 4 + src/lib.rs | 3 +- src/transformer.rs | 1523 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 1532 insertions(+), 1 deletion(-) create mode 100644 src/transformer.rs diff --git a/Cargo.toml b/Cargo.toml index fad0466..023f6fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,9 @@ description = "A complete unparser for RustPython ASTs" license-file = "LICENSE" rust-version = "1.72.1" +[features] +transformer = [] + [dependencies] rustpython-ast = { version = ">=0.4.0" } rustpython-literal = ">=0.4.0" diff --git a/README.md b/README.md index 99a5205..fd81205 100644 --- a/README.md +++ b/README.md @@ -34,3 +34,7 @@ fn main() { // ... } ``` + +# Transformer + +This crate also contains a transformer trait for easy transformation of ASTs with the possibility of removing nodes. Enable the `transformer` feature to use it. It is similar to rustpython-ast's visitor, with the difference that a visit functions always return `Option<...>` and statements/expressions are passed as mutable. diff --git a/src/lib.rs b/src/lib.rs index 85c9324..93dd009 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,8 @@ pub mod unparser; mod utils; pub use crate::unparser::Unparser; - +#[cfg(feature = "transformer")] +pub mod transformer; #[cfg(test)] mod tests { use super::*; diff --git a/src/transformer.rs b/src/transformer.rs new file mode 100644 index 0000000..ea33cd2 --- /dev/null +++ b/src/transformer.rs @@ -0,0 +1,1523 @@ +use rustpython_ast::{ + text_size::TextRange, Alias, Arg, ArgWithDefault, Arguments, Comprehension, ExceptHandler, + ExceptHandlerExceptHandler, Expr, ExprAttribute, ExprAwait, ExprBinOp, ExprBoolOp, ExprCall, + ExprCompare, ExprConstant, ExprDict, ExprDictComp, ExprFormattedValue, ExprGeneratorExp, + ExprIfExp, ExprJoinedStr, ExprLambda, ExprList, ExprListComp, ExprName, ExprNamedExpr, ExprSet, + ExprSetComp, ExprSlice, ExprStarred, ExprSubscript, ExprTuple, ExprUnaryOp, ExprYield, + ExprYieldFrom, Keyword, MatchCase, Pattern, PatternMatchAs, PatternMatchClass, + PatternMatchMapping, PatternMatchOr, PatternMatchSequence, PatternMatchSingleton, + PatternMatchStar, PatternMatchValue, Stmt, StmtAnnAssign, StmtAssert, StmtAssign, StmtAsyncFor, + StmtAsyncFunctionDef, StmtAsyncWith, StmtAugAssign, StmtBreak, StmtClassDef, StmtContinue, + StmtDelete, StmtExpr, StmtFor, StmtFunctionDef, StmtGlobal, StmtIf, StmtImport, StmtImportFrom, + StmtMatch, StmtNonlocal, StmtPass, StmtRaise, StmtReturn, StmtTry, StmtTryStar, StmtTypeAlias, + StmtWhile, StmtWith, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, + WithItem, +}; + +fn box_expr_option(expr: Option) -> Option> { + expr.map(|value| Box::new(value)) +} +#[allow(unused_mut)] +pub trait Transformer { + fn visit_stmt_vec(&mut self, stmts: Vec) -> Vec { + let mut new_stmts: Vec = Vec::new(); + + for stmt in stmts { + if let Some(new_stmt) = self.visit_stmt(stmt) { + new_stmts.push(new_stmt); + } + } + + return new_stmts; + } + + fn visit_stmt(&mut self, mut stmt: Stmt) -> Option { + self.generic_visit_stmt(stmt) + } + + fn generic_visit_stmt(&mut self, mut stmt: Stmt) -> Option { + match stmt { + Stmt::Delete(del) => self + .visit_stmt_delete(del) + .map(|new_stmt| Stmt::Delete(new_stmt)), + Stmt::Assert(assert) => self + .visit_stmt_assert(assert) + .map(|new_stmt| Stmt::Assert(new_stmt)), + Stmt::AnnAssign(ann_assign) => self + .visit_stmt_ann_assign(ann_assign) + .map(|new_stmt| Stmt::AnnAssign(new_stmt)), + Stmt::For(for_) => self + .visit_stmt_for(for_) + .map(|new_stmt| Stmt::For(new_stmt)), + Stmt::AsyncFor(async_for) => self + .visit_stmt_async_for(async_for) + .map(|new_stmt| Stmt::AsyncFor(new_stmt)), + Stmt::FunctionDef(func) => self + .visit_stmt_function_def(func) + .map(|new_stmt| Stmt::FunctionDef(new_stmt)), + Stmt::AsyncFunctionDef(async_func) => self + .visit_stmt_async_function_def(async_func) + .map(|new_stmt| Stmt::AsyncFunctionDef(new_stmt)), + Stmt::AsyncWith(async_with) => self + .visit_stmt_async_with(async_with) + .map(|new_stmt| Stmt::AsyncWith(new_stmt)), + Stmt::With(with) => self + .visit_stmt_with(with) + .map(|new_stmt| Stmt::With(new_stmt)), + Stmt::Break(break_) => self + .visit_stmt_break(break_) + .map(|new_stmt| Stmt::Break(new_stmt)), + Stmt::Pass(pass) => self + .visit_stmt_pass(pass) + .map(|new_stmt| Stmt::Pass(new_stmt)), + Stmt::Continue(continue_) => self + .visit_stmt_continue(continue_) + .map(|new_stmt| Stmt::Continue(new_stmt)), + Stmt::Return(return_) => self + .visit_stmt_return(return_) + .map(|new_stmt| Stmt::Return(new_stmt)), + Stmt::Raise(raise) => self + .visit_stmt_raise(raise) + .map(|new_stmt| Stmt::Raise(new_stmt)), + Stmt::ClassDef(stmt_class_def) => self + .visit_stmt_class_def(stmt_class_def) + .map(|new_stmt| Stmt::ClassDef(new_stmt)), + Stmt::Assign(stmt_assign) => self + .visit_stmt_assign(stmt_assign) + .map(|new_stmt| Stmt::Assign(new_stmt)), + Stmt::TypeAlias(stmt_type_alias) => self + .visit_stmt_type_alias(stmt_type_alias) + .map(|new_stmt| Stmt::TypeAlias(new_stmt)), + Stmt::AugAssign(stmt_aug_assign) => self + .visit_stmt_aug_assign(stmt_aug_assign) + .map(|new_stmt| Stmt::AugAssign(new_stmt)), + Stmt::While(stmt_while) => self + .visit_stmt_while(stmt_while) + .map(|new_stmt| Stmt::While(new_stmt)), + Stmt::If(stmt_if) => self + .visit_stmt_if(stmt_if) + .map(|new_stmt| Stmt::If(new_stmt)), + Stmt::Match(stmt_match) => self + .visit_stmt_match(stmt_match) + .map(|new_stmt| Stmt::Match(new_stmt)), + Stmt::Try(stmt_try) => self + .visit_stmt_try(stmt_try) + .map(|new_stmt| Stmt::Try(new_stmt)), + Stmt::TryStar(stmt_try_star) => self + .visit_stmt_try_star(stmt_try_star) + .map(|new_stmt| Stmt::TryStar(new_stmt)), + Stmt::Import(stmt_import) => self + .visit_stmt_import(stmt_import) + .map(|new_stmt| Stmt::Import(new_stmt)), + Stmt::ImportFrom(stmt_import_from) => self + .visit_stmt_import_from(stmt_import_from) + .map(|new_stmt| Stmt::ImportFrom(new_stmt)), + Stmt::Global(stmt_global) => self + .visit_stmt_global(stmt_global) + .map(|new_stmt| Stmt::Global(new_stmt)), + Stmt::Nonlocal(stmt_nonlocal) => self + .visit_stmt_nonlocal(stmt_nonlocal) + .map(|new_stmt| Stmt::Nonlocal(new_stmt)), + Stmt::Expr(stmt_expr) => self + .visit_stmt_expr(stmt_expr) + .map(|new_stmt| Stmt::Expr(new_stmt)), + } + } + + fn generic_visit_keyword_vec(&mut self, mut keywords: Vec) -> Vec { + let mut new_keywords = Vec::new(); + + for keyword in keywords { + if let Some(new_keyword) = self.visit_keyword(keyword) { + new_keywords.push(new_keyword); + } + } + new_keywords + } + + fn visit_keyword(&mut self, mut keyword: Keyword) -> Option { + self.generic_visit_keyword(keyword) + } + + fn generic_visit_keyword(&mut self, mut keyword: Keyword) -> Option { + keyword.value = self + .visit_expr(keyword.value) + .expect("Cannot remove value from keyword"); + + Some(keyword) + } + + fn visit_stmt_class_def(&mut self, mut stmt: StmtClassDef) -> Option { + self.generic_visit_stmt_class_def(stmt) + } + + fn generic_visit_stmt_class_def(&mut self, mut stmt: StmtClassDef) -> Option { + stmt.decorator_list = self.visit_expr_vec(stmt.decorator_list); + + stmt.type_params = self.generic_visit_type_param_vec(stmt.type_params); + stmt.bases = self.visit_expr_vec(stmt.bases); + stmt.keywords = self.generic_visit_keyword_vec(stmt.keywords); + stmt.body = self.visit_stmt_vec(stmt.body); + + if stmt.body.len() == 0 { + stmt.body.push(Stmt::Pass(StmtPass { + range: TextRange::default(), + })); + } + + Some(stmt) + } + + fn visit_stmt_assign(&mut self, mut stmt: StmtAssign) -> Option { + self.generic_visit_stmt_assign(stmt) + } + + fn generic_visit_stmt_assign(&mut self, mut stmt: StmtAssign) -> Option { + stmt.targets = self.visit_expr_vec(stmt.targets); + if stmt.targets.len() == 0 { + panic!("Cannot remove all targets from assignment") + } + stmt.value = Box::new( + self.visit_expr(*stmt.value) + .expect("Cannot remove value from assignment"), + ); + + Some(stmt) + } + + fn visit_stmt_type_alias(&mut self, mut stmt: StmtTypeAlias) -> Option { + self.generic_visit_stmt_type_alias(stmt) + } + + fn generic_visit_stmt_type_alias(&mut self, mut stmt: StmtTypeAlias) -> Option { + stmt.name = Box::new( + self.visit_expr(*stmt.name) + .expect("Cannot remove name from type alias"), + ); + stmt.type_params = self.generic_visit_type_param_vec(stmt.type_params); + stmt.value = Box::new( + self.visit_expr(*stmt.value) + .expect("Cannot remove value from type alias"), + ); + + Some(stmt) + } + + fn visit_stmt_aug_assign(&mut self, mut stmt: StmtAugAssign) -> Option { + self.generic_visit_stmt_aug_assign(stmt) + } + + fn generic_visit_stmt_aug_assign(&mut self, mut stmt: StmtAugAssign) -> Option { + stmt.value = Box::new( + self.visit_expr(*stmt.value) + .expect("Cannot remove value from augmented assignment"), + ); + stmt.target = Box::new( + self.visit_expr(*stmt.target) + .expect("Cannot remove target from augmented assignment"), + ); + + Some(stmt) + } + + fn visit_stmt_while(&mut self, mut stmt: StmtWhile) -> Option { + self.generic_visit_stmt_while(stmt) + } + + fn generic_visit_stmt_while(&mut self, mut stmt: StmtWhile) -> Option { + stmt.test = Box::new( + self.visit_expr(*stmt.test) + .expect("Cannot remove test from while statement"), + ); + stmt.body = self.visit_stmt_vec(stmt.body); + stmt.orelse = self.visit_stmt_vec(stmt.orelse); + + if stmt.body.len() == 0 && stmt.orelse.len() == 0 { + return None; + } + + Some(stmt) + } + + fn visit_stmt_if(&mut self, mut stmt: StmtIf) -> Option { + self.generic_visit_stmt_if(stmt) + } + + fn generic_visit_stmt_if(&mut self, mut stmt: StmtIf) -> Option { + stmt.test = Box::new( + self.visit_expr(*stmt.test) + .expect("Cannot remove test from if statement"), + ); + stmt.body = self.visit_stmt_vec(stmt.body); + stmt.orelse = self.visit_stmt_vec(stmt.orelse); + + if stmt.body.len() == 0 && stmt.orelse.len() == 0 { + return None; + } + + Some(stmt) + } + + fn visit_pattern_match_or(&mut self, mut pattern: PatternMatchOr) -> Option { + self.generic_visit_pattern_match_or(pattern) + } + + fn generic_visit_pattern_match_or( + &mut self, + mut pattern: PatternMatchOr, + ) -> Option { + pattern.patterns = self.generic_visit_pattern_vec(pattern.patterns); + if pattern.patterns.len() == 0 { + return None; + } + + Some(pattern) + } + + fn visit_pattern_match_as(&mut self, mut pattern: PatternMatchAs) -> Option { + self.generic_visit_pattern_match_as(pattern) + } + + fn generic_visit_pattern_match_as( + &mut self, + mut pattern: PatternMatchAs, + ) -> Option { + if let Some(inner_pattern) = pattern.pattern { + pattern.pattern = self + .visit_pattern(*inner_pattern) + .map(|new_pattern| Box::new(new_pattern)); + } + + Some(pattern) + } + + fn visit_pattern_match_mapping( + &mut self, + mut pattern: PatternMatchMapping, + ) -> Option { + self.generic_visit_pattern_match_mapping(pattern) + } + + fn generic_visit_pattern_match_mapping( + &mut self, + mut pattern: PatternMatchMapping, + ) -> Option { + pattern.keys = self.visit_expr_vec(pattern.keys); + pattern.patterns = self.generic_visit_pattern_vec(pattern.patterns); + Some(pattern) + } + + fn visit_pattern_match_star( + &mut self, + mut pattern: PatternMatchStar, + ) -> Option { + self.generic_visit_pattern_match_star(pattern) + } + + fn generic_visit_pattern_match_star( + &mut self, + mut pattern: PatternMatchStar, + ) -> Option { + Some(pattern) + } + + fn visit_pattern_match_class( + &mut self, + mut pattern: PatternMatchClass, + ) -> Option { + self.generic_visit_pattern_match_class(pattern) + } + + fn generic_visit_pattern_match_class( + &mut self, + mut pattern: PatternMatchClass, + ) -> Option { + pattern.cls = Box::new( + self.visit_expr(*pattern.cls) + .expect("Cannot remove class from pattern match class"), + ); + pattern.patterns = self.generic_visit_pattern_vec(pattern.patterns); + pattern.kwd_patterns = self.generic_visit_pattern_vec(pattern.kwd_patterns); + Some(pattern) + } + + fn visit_pattern_match_sequence( + &mut self, + mut pattern: PatternMatchSequence, + ) -> Option { + self.generic_visit_pattern_match_sequence(pattern) + } + + fn generic_visit_pattern_match_sequence( + &mut self, + mut pattern: PatternMatchSequence, + ) -> Option { + pattern.patterns = self.generic_visit_pattern_vec(pattern.patterns); + if pattern.patterns.len() == 0 { + return None; + } + + Some(pattern) + } + + fn visit_pattern_match_singleton( + &mut self, + mut pattern: PatternMatchSingleton, + ) -> Option { + self.generic_visit_pattern_match_singleton(pattern) + } + + fn generic_visit_pattern_match_singleton( + &mut self, + mut pattern: PatternMatchSingleton, + ) -> Option { + Some(pattern) + } + + fn visit_pattern_match_value( + &mut self, + mut pattern: PatternMatchValue, + ) -> Option { + self.generic_visit_pattern_match_value(pattern) + } + + fn generic_visit_pattern_match_value( + &mut self, + mut pattern: PatternMatchValue, + ) -> Option { + pattern.value = Box::new( + self.visit_expr(*pattern.value) + .expect("Cannot remove value from pattern match value"), + ); + Some(pattern) + } + + fn generic_visit_pattern_vec(&mut self, patterns: Vec) -> Vec { + let mut new_patterns: Vec = Vec::new(); + for pattern in patterns { + if let Some(new_pattern) = self.visit_pattern(pattern) { + new_patterns.push(new_pattern); + } + } + + new_patterns + } + + fn visit_pattern(&mut self, pattern: Pattern) -> Option { + self.generic_visit_pattern(pattern) + } + + fn generic_visit_pattern(&mut self, pattern: Pattern) -> Option { + match pattern { + Pattern::MatchValue(pattern_match_value) => self + .visit_pattern_match_value(pattern_match_value) + .map(|new_pattern| Pattern::MatchValue(new_pattern)), + Pattern::MatchSingleton(pattern_match_singleton) => self + .visit_pattern_match_singleton(pattern_match_singleton) + .map(|new_pattern| Pattern::MatchSingleton(new_pattern)), + Pattern::MatchSequence(pattern_match_sequence) => self + .visit_pattern_match_sequence(pattern_match_sequence) + .map(|new_pattern| Pattern::MatchSequence(new_pattern)), + Pattern::MatchMapping(pattern_match_mapping) => self + .visit_pattern_match_mapping(pattern_match_mapping) + .map(|new_pattern| Pattern::MatchMapping(new_pattern)), + Pattern::MatchClass(pattern_match_class) => self + .visit_pattern_match_class(pattern_match_class) + .map(|new_pattern| Pattern::MatchClass(new_pattern)), + Pattern::MatchStar(pattern_match_star) => self + .visit_pattern_match_star(pattern_match_star) + .map(|new_pattern| Pattern::MatchStar(new_pattern)), + Pattern::MatchAs(pattern_match_as) => self + .visit_pattern_match_as(pattern_match_as) + .map(|new_pattern| Pattern::MatchAs(new_pattern)), + Pattern::MatchOr(pattern_match_or) => self + .visit_pattern_match_or(pattern_match_or) + .map(|new_pattern| Pattern::MatchOr(new_pattern)), + } + } + + fn generic_visit_match_case_vec(&mut self, cases: Vec) -> Vec { + let mut new_cases: Vec = Vec::new(); + for case in cases { + if let Some(new_case) = self.visit_match_case(case) { + new_cases.push(new_case); + } + } + + new_cases + } + + fn visit_match_case(&mut self, mut case: MatchCase) -> Option { + self.generic_visit_match_case(case) + } + + fn generic_visit_match_case(&mut self, mut case: MatchCase) -> Option { + case.pattern = self + .visit_pattern(case.pattern) + .expect("Cannot remove pattern from match case"); + if let Some(guard) = case.guard { + case.guard = box_expr_option(self.visit_expr(*guard)); + } + + case.body = self.visit_stmt_vec(case.body); + if case.body.len() == 0 { + return None; + } + Some(case) + } + + fn visit_stmt_match(&mut self, mut stmt: StmtMatch) -> Option { + self.generic_visit_stmt_match(stmt) + } + + fn generic_visit_stmt_match(&mut self, mut stmt: StmtMatch) -> Option { + stmt.subject = Box::new( + self.visit_expr(*stmt.subject) + .expect("Cannot remove subject from match statement"), + ); + stmt.cases = self.generic_visit_match_case_vec(stmt.cases); + if stmt.cases.len() == 0 { + return None; + } + Some(stmt) + } + + fn generic_visit_except_handler_vec( + &mut self, + handlers: Vec, + ) -> Vec { + let mut new_handlers: Vec = Vec::new(); + + for handler in handlers { + if let Some(new_handler) = self.visit_except_handler(handler) { + new_handlers.push(new_handler); + } + } + + new_handlers + } + + fn visit_except_handler(&mut self, mut handler: ExceptHandler) -> Option { + self.generic_visit_except_handler(handler) + } + + fn generic_visit_except_handler( + &mut self, + mut handler: ExceptHandler, + ) -> Option { + match handler { + ExceptHandler::ExceptHandler(except_handler) => self + .visit_except_handler_except_handler(except_handler) + .map(|new_except_handler| ExceptHandler::ExceptHandler(new_except_handler)), + } + } + + fn visit_except_handler_except_handler( + &mut self, + mut except_handler: ExceptHandlerExceptHandler, + ) -> Option { + self.generic_visit_except_handler_except_handler(except_handler) + } + + fn generic_visit_except_handler_except_handler( + &mut self, + mut except_handler: ExceptHandlerExceptHandler, + ) -> Option { + except_handler.body = self.visit_stmt_vec(except_handler.body); + if except_handler.body.len() == 0 { + return None; + } + Some(except_handler) + } + + fn visit_stmt_try(&mut self, mut stmt: StmtTry) -> Option { + self.generic_visit_stmt_try(stmt) + } + + fn generic_visit_stmt_try(&mut self, mut stmt: StmtTry) -> Option { + stmt.body = self.visit_stmt_vec(stmt.body); + stmt.finalbody = self.visit_stmt_vec(stmt.finalbody); + stmt.handlers = self.generic_visit_except_handler_vec(stmt.handlers); + stmt.orelse = self.visit_stmt_vec(stmt.orelse); + + if stmt.body.len() == 0 { + return None; + } + + Some(stmt) + } + + fn visit_stmt_try_star(&mut self, mut stmt: StmtTryStar) -> Option { + self.generic_visit_stmt_try_star(stmt) + } + + fn generic_visit_stmt_try_star(&mut self, mut stmt: StmtTryStar) -> Option { + stmt.body = self.visit_stmt_vec(stmt.body); + stmt.finalbody = self.visit_stmt_vec(stmt.finalbody); + stmt.handlers = self.generic_visit_except_handler_vec(stmt.handlers); + stmt.orelse = self.visit_stmt_vec(stmt.orelse); + + if stmt.body.len() == 0 { + return None; + } + + Some(stmt) + } + + fn generic_visit_alias_vec(&mut self, aliases: Vec) -> Vec { + let mut new_aliases: Vec = Vec::new(); + + for alias in aliases { + if let Some(new_alias) = self.visit_alias(alias) { + new_aliases.push(new_alias); + } + } + + return new_aliases; + } + + fn visit_alias(&mut self, mut alias: Alias) -> Option { + self.generic_visit_alias(alias) + } + + fn generic_visit_alias(&mut self, mut alias: Alias) -> Option { + Some(alias) + } + + fn visit_stmt_import(&mut self, mut stmt: StmtImport) -> Option { + self.generic_visit_stmt_import(stmt) + } + + fn generic_visit_stmt_import(&mut self, mut stmt: StmtImport) -> Option { + stmt.names = self.generic_visit_alias_vec(stmt.names); + + if stmt.names.len() == 0 { + return None; + } + Some(stmt) + } + + fn visit_stmt_import_from(&mut self, mut stmt: StmtImportFrom) -> Option { + self.generic_visit_stmt_import_from(stmt) + } + + fn generic_visit_stmt_import_from( + &mut self, + mut stmt: StmtImportFrom, + ) -> Option { + stmt.names = self.generic_visit_alias_vec(stmt.names); + + if stmt.names.len() == 0 { + return None; + } + Some(stmt) + } + + fn visit_stmt_global(&mut self, mut stmt: StmtGlobal) -> Option { + self.generic_visit_stmt_global(stmt) + } + + fn generic_visit_stmt_global(&mut self, mut stmt: StmtGlobal) -> Option { + if stmt.names.len() == 0 { + return None; + } + Some(stmt) + } + + fn visit_stmt_nonlocal(&mut self, mut stmt: StmtNonlocal) -> Option { + self.generic_visit_stmt_nonlocal(stmt) + } + + fn generic_visit_stmt_nonlocal(&mut self, mut stmt: StmtNonlocal) -> Option { + if stmt.names.len() == 0 { + return None; + } + Some(stmt) + } + + fn visit_stmt_expr(&mut self, mut stmt: StmtExpr) -> Option { + self.generic_visit_stmt_expr(stmt) + } + + fn generic_visit_stmt_expr(&mut self, mut stmt: StmtExpr) -> Option { + match self.visit_expr(*stmt.value) { + Some(new_expr) => { + stmt.value = Box::new(new_expr); + return Some(stmt); + } + None => None, + } + } + + fn visit_stmt_raise(&mut self, mut stmt: StmtRaise) -> Option { + self.generic_visit_stmt_raise(stmt) + } + + fn generic_visit_stmt_raise(&mut self, mut stmt: StmtRaise) -> Option { + if let Some(exc) = stmt.exc { + stmt.exc = box_expr_option(self.visit_expr(*exc)); + } + + if let Some(cause) = stmt.cause { + stmt.cause = box_expr_option(self.visit_expr(*cause)); + } + + Some(stmt) + } + + fn visit_stmt_return(&mut self, mut stmt: StmtReturn) -> Option { + self.generic_visit_stmt_return(stmt) + } + + fn generic_visit_stmt_return(&mut self, mut stmt: StmtReturn) -> Option { + if let Some(value) = stmt.value { + stmt.value = box_expr_option(self.visit_expr(*value)); + } + + Some(stmt) + } + + fn visit_stmt_continue(&mut self, mut stmt: StmtContinue) -> Option { + self.generic_visit_stmt_continue(stmt) + } + + fn generic_visit_stmt_continue(&mut self, mut stmt: StmtContinue) -> Option { + Some(stmt) + } + + fn visit_stmt_pass(&mut self, mut stmt: StmtPass) -> Option { + self.generic_visit_stmt_pass(stmt) + } + + fn generic_visit_stmt_pass(&mut self, mut stmt: StmtPass) -> Option { + Some(stmt) + } + + fn visit_stmt_break(&mut self, mut stmt: StmtBreak) -> Option { + self.generic_visit_stmt_break(stmt) + } + + fn generic_visit_stmt_break(&mut self, mut stmt: StmtBreak) -> Option { + Some(stmt) + } + + fn visit_stmt_with(&mut self, mut stmt: StmtWith) -> Option { + self.generic_visit_stmt_with(stmt) + } + + fn generic_visit_stmt_with(&mut self, mut stmt: StmtWith) -> Option { + stmt.items = self.generic_visit_with_item_vec(stmt.items); + stmt.body = self.visit_stmt_vec(stmt.body); + + if stmt.body.len() == 0 { + return None; + } + + Some(stmt) + } + + fn visit_stmt_async_with(&mut self, mut stmt: StmtAsyncWith) -> Option { + self.generic_visit_stmt_async_with(stmt) + } + + fn generic_visit_stmt_async_with(&mut self, mut stmt: StmtAsyncWith) -> Option { + stmt.items = self.generic_visit_with_item_vec(stmt.items); + stmt.body = self.visit_stmt_vec(stmt.body); + if stmt.body.len() == 0 { + return None; + } + Some(stmt) + } + + fn visit_stmt_function_def(&mut self, mut stmt: StmtFunctionDef) -> Option { + self.generic_visit_stmt_function_def(stmt) + } + + fn generic_visit_stmt_function_def( + &mut self, + mut stmt: StmtFunctionDef, + ) -> Option { + stmt.type_params = self.generic_visit_type_param_vec(stmt.type_params); + stmt.decorator_list = self.visit_expr_vec(stmt.decorator_list); + stmt.args = Box::new(self.visit_arguments(*stmt.args)); + if let Some(returns) = stmt.returns { + stmt.returns = box_expr_option(self.visit_expr(*returns)); + } + stmt.body = self.visit_stmt_vec(stmt.body); + if stmt.body.len() == 0 { + return None; + } + Some(stmt) + } + + fn visit_stmt_async_function_def( + &mut self, + mut stmt: StmtAsyncFunctionDef, + ) -> Option { + self.generic_visit_stmt_async_function_def(stmt) + } + + fn generic_visit_stmt_async_function_def( + &mut self, + mut stmt: StmtAsyncFunctionDef, + ) -> Option { + stmt.type_params = self.generic_visit_type_param_vec(stmt.type_params); + stmt.decorator_list = self.visit_expr_vec(stmt.decorator_list); + stmt.args = Box::new(self.visit_arguments(*stmt.args)); + if let Some(returns) = stmt.returns { + stmt.returns = box_expr_option(self.visit_expr(*returns)); + } + stmt.body = self.visit_stmt_vec(stmt.body); + if stmt.body.len() == 0 { + return None; + } + Some(stmt) + } + + fn visit_stmt_for(&mut self, mut stmt: StmtFor) -> Option { + self.generic_visit_for(stmt) + } + + fn generic_visit_for(&mut self, mut stmt: StmtFor) -> Option { + stmt.body = self.visit_stmt_vec(stmt.body); + stmt.iter = Box::new( + self.visit_expr(*stmt.iter) + .expect("Cannot remove iter from async for"), + ); + stmt.orelse = self.visit_stmt_vec(stmt.orelse); + stmt.target = Box::new( + self.visit_expr(*stmt.target) + .expect("Cannot remove target from async for"), + ); + if stmt.body.len() == 0 { + return None; + } + Some(stmt) + } + + fn visit_stmt_async_for(&mut self, mut stmt: StmtAsyncFor) -> Option { + self.generic_visit_async_for(stmt) + } + + fn generic_visit_async_for(&mut self, mut stmt: StmtAsyncFor) -> Option { + stmt.body = self.visit_stmt_vec(stmt.body); + stmt.iter = Box::new( + self.visit_expr(*stmt.iter) + .expect("Cannot remove iter from async for"), + ); + stmt.orelse = self.visit_stmt_vec(stmt.orelse); + stmt.target = Box::new( + self.visit_expr(*stmt.target) + .expect("Cannot remove target from async for"), + ); + if stmt.body.len() == 0 { + return None; + } + Some(stmt) + } + + fn visit_stmt_ann_assign(&mut self, mut stmt: StmtAnnAssign) -> Option { + self.generic_visit_ann_assign(stmt) + } + + fn generic_visit_ann_assign(&mut self, mut stmt: StmtAnnAssign) -> Option { + stmt.annotation = Box::new( + self.visit_expr(*stmt.annotation) + .expect("Cannot remove annotation from annotated assignment"), + ); + + stmt.target = Box::new( + self.visit_expr(*stmt.target) + .expect("Cannot remove target from annotated assignment"), + ); + + if let Some(value) = stmt.value { + stmt.value = box_expr_option(self.visit_expr(*value)); + } + + Some(stmt) + } + + fn visit_stmt_assert(&mut self, mut stmt: StmtAssert) -> Option { + self.generic_visit_assert(stmt) + } + + fn generic_visit_assert(&mut self, mut stmt: StmtAssert) -> Option { + if let Some(msg) = stmt.msg { + stmt.msg = box_expr_option(self.visit_expr(*msg)); + } + + stmt.test = Box::new( + self.visit_expr(*stmt.test) + .expect("Assertion test cannot be removed"), + ); + + Some(stmt) + } + + fn visit_stmt_delete(&mut self, mut stmt: StmtDelete) -> Option { + self.generic_visit_delete(stmt) + } + + fn generic_visit_delete(&mut self, mut stmt: StmtDelete) -> Option { + stmt.targets = self.visit_expr_vec(stmt.targets); + if stmt.targets.len() == 0 { + return None; + } + Some(stmt) + } + + fn visit_expr_vec(&mut self, exprs: Vec) -> Vec { + let mut new_exprs: Vec = Vec::new(); + + for expr in exprs { + if let Some(new_expr) = self.visit_expr(expr) { + new_exprs.push(new_expr); + } + } + + return new_exprs; + } + + fn visit_expr(&mut self, expr: Expr) -> Option { + self.generic_visit_expr(expr) + } + + fn generic_visit_expr(&mut self, expr: Expr) -> Option { + match expr { + Expr::BoolOp(expr_bool_op) => self + .visit_expr_bool_op(expr_bool_op) + .map(|new_expr| Expr::BoolOp(new_expr)), + Expr::NamedExpr(expr_named_expr) => self + .visit_expr_named_expr(expr_named_expr) + .map(|new_expr| Expr::NamedExpr(new_expr)), + Expr::BinOp(expr_bin_op) => self + .visit_expr_bin_op(expr_bin_op) + .map(|new_expr| Expr::BinOp(new_expr)), + Expr::UnaryOp(expr_unary_op) => self + .visit_expr_unary_op(expr_unary_op) + .map(|new_expr| Expr::UnaryOp(new_expr)), + Expr::Lambda(expr_lambda) => self + .visit_expr_lambda(expr_lambda) + .map(|new_expr| Expr::Lambda(new_expr)), + Expr::IfExp(expr_if_exp) => self + .visit_expr_if_exp(expr_if_exp) + .map(|new_expr| Expr::IfExp(new_expr)), + Expr::Dict(expr_dict) => self + .visit_expr_dict(expr_dict) + .map(|new_expr| Expr::Dict(new_expr)), + Expr::Set(expr_set) => self + .visit_expr_set(expr_set) + .map(|new_expr| Expr::Set(new_expr)), + Expr::ListComp(expr_list_comp) => self + .visit_expr_list_comp(expr_list_comp) + .map(|new_expr| Expr::ListComp(new_expr)), + Expr::SetComp(expr_set_comp) => self + .visit_expr_set_comp(expr_set_comp) + .map(|new_expr| Expr::SetComp(new_expr)), + Expr::DictComp(expr_dict_comp) => self + .visit_expr_dict_comp(expr_dict_comp) + .map(|new_expr| Expr::DictComp(new_expr)), + Expr::GeneratorExp(expr_generator_exp) => self + .visit_expr_generator_exp(expr_generator_exp) + .map(|new_expr| Expr::GeneratorExp(new_expr)), + Expr::Await(expr_await) => self + .visit_expr_await(expr_await) + .map(|new_expr| Expr::Await(new_expr)), + Expr::Yield(expr_yield) => self + .visit_expr_yield(expr_yield) + .map(|new_expr| Expr::Yield(new_expr)), + Expr::YieldFrom(expr_yield_from) => self + .visit_expr_yield_from(expr_yield_from) + .map(|new_expr| Expr::YieldFrom(new_expr)), + Expr::Compare(expr_compare) => self + .visit_expr_compare(expr_compare) + .map(|new_expr| Expr::Compare(new_expr)), + Expr::Call(expr_call) => self + .visit_expr_call(expr_call) + .map(|new_expr| Expr::Call(new_expr)), + Expr::FormattedValue(expr_formatted_value) => self + .visit_expr_formatted_value(expr_formatted_value) + .map(|new_expr| Expr::FormattedValue(new_expr)), + Expr::JoinedStr(expr_joined_str) => self + .visit_expr_joined_str(expr_joined_str) + .map(|new_expr| Expr::JoinedStr(new_expr)), + Expr::Constant(expr_constant) => self + .visit_expr_constant(expr_constant) + .map(|new_expr| Expr::Constant(new_expr)), + Expr::Attribute(expr_attribute) => self + .visit_expr_attribute(expr_attribute) + .map(|new_expr| Expr::Attribute(new_expr)), + Expr::Subscript(expr_subscript) => self + .visit_expr_subscript(expr_subscript) + .map(|new_expr| Expr::Subscript(new_expr)), + Expr::Starred(expr_starred) => self + .visit_expr_starred(expr_starred) + .map(|new_expr| Expr::Starred(new_expr)), + Expr::Name(expr_name) => self + .visit_expr_name(expr_name) + .map(|new_expr| Expr::Name(new_expr)), + Expr::List(expr_list) => self + .visit_expr_list(expr_list) + .map(|new_expr| Expr::List(new_expr)), + Expr::Tuple(expr_tuple) => self + .visit_expr_tuple(expr_tuple) + .map(|new_expr| Expr::Tuple(new_expr)), + Expr::Slice(expr_slice) => self + .visit_expr_slice(expr_slice) + .map(|new_expr| Expr::Slice(new_expr)), + } + } + + fn visit_expr_slice(&mut self, mut expr: ExprSlice) -> Option { + self.generic_visit_expr_slice(expr) + } + + fn generic_visit_expr_slice(&mut self, mut expr: ExprSlice) -> Option { + if let Some(lower) = expr.lower { + expr.lower = box_expr_option(self.visit_expr(*lower)); + } + + if let Some(upper) = expr.upper { + expr.upper = box_expr_option(self.visit_expr(*upper)); + } + + if let Some(step) = expr.step { + expr.step = box_expr_option(self.visit_expr(*step)); + } + + Some(expr) + } + + fn visit_expr_tuple(&mut self, mut expr: ExprTuple) -> Option { + self.generic_visit_expr_tuple(expr) + } + + fn generic_visit_expr_tuple(&mut self, mut expr: ExprTuple) -> Option { + expr.elts = self.visit_expr_vec(expr.elts); + + Some(expr) + } + + fn visit_expr_list(&mut self, mut expr: ExprList) -> Option { + self.generic_visit_expr_list(expr) + } + + fn generic_visit_expr_list(&mut self, mut expr: ExprList) -> Option { + expr.elts = self.visit_expr_vec(expr.elts); + Some(expr) + } + + fn visit_expr_name(&mut self, mut expr: ExprName) -> Option { + self.generic_visit_expr_name(expr) + } + + fn generic_visit_expr_name(&mut self, mut expr: ExprName) -> Option { + Some(expr) + } + + fn visit_expr_starred(&mut self, mut expr: ExprStarred) -> Option { + self.generic_visit_expr_starred(expr) + } + + fn generic_visit_expr_starred(&mut self, mut expr: ExprStarred) -> Option { + expr.value = Box::new( + self.visit_expr(*expr.value) + .expect("Cannot remove value from starred expression"), + ); + + Some(expr) + } + + fn visit_expr_subscript(&mut self, mut expr: ExprSubscript) -> Option { + self.generic_visit_expr_subscript(expr) + } + + fn generic_visit_expr_subscript(&mut self, mut expr: ExprSubscript) -> Option { + expr.value = Box::new( + self.visit_expr(*expr.value) + .expect("Cannot remove value from subscript expression"), + ); + expr.slice = Box::new( + self.visit_expr(*expr.slice) + .expect("Cannot remove slice from subscript expression"), + ); + Some(expr) + } + + fn visit_expr_attribute(&mut self, mut expr: ExprAttribute) -> Option { + self.generic_visit_expr_attribute(expr) + } + + fn generic_visit_expr_attribute(&mut self, mut expr: ExprAttribute) -> Option { + expr.value = Box::new( + self.visit_expr(*expr.value) + .expect("Cannot remove value from attribute expression"), + ); + Some(expr) + } + + fn visit_expr_constant(&mut self, mut expr: ExprConstant) -> Option { + self.generic_visit_expr_constant(expr) + } + + fn generic_visit_expr_constant(&mut self, mut expr: ExprConstant) -> Option { + Some(expr) + } + + fn visit_expr_joined_str(&mut self, mut expr: ExprJoinedStr) -> Option { + self.generic_visit_expr_joined_str(expr) + } + + fn generic_visit_expr_joined_str(&mut self, mut expr: ExprJoinedStr) -> Option { + expr.values = self.visit_expr_vec(expr.values); + + Some(expr) + } + + fn visit_expr_formatted_value( + &mut self, + mut expr: ExprFormattedValue, + ) -> Option { + self.generic_visit_expr_formatted_value(expr) + } + + fn generic_visit_expr_formatted_value( + &mut self, + mut expr: ExprFormattedValue, + ) -> Option { + expr.value = Box::new( + self.visit_expr(*expr.value) + .expect("Cannot remove value from formatted value expression"), + ); + if let Some(format_spec) = expr.format_spec { + expr.format_spec = box_expr_option(self.visit_expr(*format_spec)); + } + + Some(expr) + } + + fn visit_expr_call(&mut self, mut expr: ExprCall) -> Option { + self.generic_visit_expr_call(expr) + } + + fn generic_visit_expr_call(&mut self, mut expr: ExprCall) -> Option { + expr.func = Box::new( + self.visit_expr(*expr.func) + .expect("Cannot remove func from call expression"), + ); + expr.args = self.visit_expr_vec(expr.args); + expr.keywords = self.generic_visit_keyword_vec(expr.keywords); + Some(expr) + } + + fn visit_expr_compare(&mut self, mut expr: ExprCompare) -> Option { + self.generic_visit_expr_compare(expr) + } + + fn generic_visit_expr_compare(&mut self, mut expr: ExprCompare) -> Option { + expr.left = Box::new( + self.visit_expr(*expr.left) + .expect("Cannot remove left from compare expression"), + ); + expr.comparators = self.visit_expr_vec(expr.comparators); + Some(expr) + } + + fn visit_expr_yield_from(&mut self, mut expr: ExprYieldFrom) -> Option { + self.generic_visit_expr_yield_from(expr) + } + + fn generic_visit_expr_yield_from(&mut self, mut expr: ExprYieldFrom) -> Option { + expr.value = Box::new( + self.visit_expr(*expr.value) + .expect("Cannot remove value from yield from expression"), + ); + Some(expr) + } + + fn visit_expr_yield(&mut self, mut expr: ExprYield) -> Option { + self.generic_visit_expr_yield(expr) + } + + fn generic_visit_expr_yield(&mut self, mut expr: ExprYield) -> Option { + if let Some(value) = expr.value { + expr.value = box_expr_option(self.visit_expr(*value)); + } + + Some(expr) + } + + fn visit_expr_await(&mut self, mut expr: ExprAwait) -> Option { + self.generic_visit_expr_await(expr) + } + + fn generic_visit_expr_await(&mut self, mut expr: ExprAwait) -> Option { + match self.visit_expr(*expr.value) { + Some(new_value) => { + expr.value = Box::new(new_value); + Some(expr) + } + None => None, + } + } + + fn generic_visit_comprehension_vec(&mut self, comps: Vec) -> Vec { + let mut new_comps: Vec = Vec::new(); + + for comp in comps { + if let Some(new_comp) = self.visit_comprehension(comp) { + new_comps.push(new_comp); + } + } + new_comps + } + + fn visit_comprehension(&mut self, mut comp: Comprehension) -> Option { + self.generic_visit_comprehension(comp) + } + + fn generic_visit_comprehension(&mut self, mut comp: Comprehension) -> Option { + comp.ifs = self.visit_expr_vec(comp.ifs); + comp.iter = self + .visit_expr(comp.iter) + .expect("Cannot remove iter from comprehension"); + comp.target = self + .visit_expr(comp.target) + .expect("Cannot remove target from comprehension"); + + Some(comp) + } + + fn visit_expr_generator_exp(&mut self, mut expr: ExprGeneratorExp) -> Option { + self.generic_visit_expr_generator_expr(expr) + } + + fn generic_visit_expr_generator_expr( + &mut self, + mut expr: ExprGeneratorExp, + ) -> Option { + expr.elt = Box::new( + self.visit_expr(*expr.elt) + .expect("Cannot remove elt from generator expression"), + ); + expr.generators = self.generic_visit_comprehension_vec(expr.generators); + Some(expr) + } + + fn visit_expr_dict_comp(&mut self, mut expr: ExprDictComp) -> Option { + self.generic_visit_expr_dict_comp(expr) + } + + fn generic_visit_expr_dict_comp(&mut self, mut expr: ExprDictComp) -> Option { + expr.key = Box::new( + self.visit_expr(*expr.key) + .expect("Cannot remove key from dict comprehension"), + ); + expr.value = Box::new( + self.visit_expr(*expr.value) + .expect("Cannot remove value from dict comprehension"), + ); + expr.generators = self.generic_visit_comprehension_vec(expr.generators); + Some(expr) + } + + fn visit_expr_set_comp(&mut self, mut expr: ExprSetComp) -> Option { + self.generic_visit_expr_set_comp(expr) + } + + fn generic_visit_expr_set_comp(&mut self, mut expr: ExprSetComp) -> Option { + expr.elt = Box::new( + self.visit_expr(*expr.elt) + .expect("Cannot remove elt from set comprehension"), + ); + expr.generators = self.generic_visit_comprehension_vec(expr.generators); + Some(expr) + } + + fn visit_expr_list_comp(&mut self, mut expr: ExprListComp) -> Option { + self.generic_visit_expr_list_comp(expr) + } + + fn generic_visit_expr_list_comp(&mut self, mut expr: ExprListComp) -> Option { + expr.elt = Box::new( + self.visit_expr(*expr.elt) + .expect("Cannot remove elt from list comprehension"), + ); + expr.generators = self.generic_visit_comprehension_vec(expr.generators); + Some(expr) + } + + fn visit_expr_set(&mut self, mut expr: ExprSet) -> Option { + self.generic_visit_expr_set(expr) + } + + fn generic_visit_expr_set(&mut self, mut expr: ExprSet) -> Option { + expr.elts = self.visit_expr_vec(expr.elts); + Some(expr) + } + + fn visit_expr_dict(&mut self, mut expr: ExprDict) -> Option { + self.generic_visit_expr_dict(expr) + } + + fn generic_visit_expr_dict(&mut self, mut expr: ExprDict) -> Option { + let mut new_keys: Vec> = Vec::new(); + for key in expr.keys { + if let Some(key_value) = key { + new_keys.push(self.visit_expr(key_value)); + } + } + expr.keys = new_keys; + expr.values = self.visit_expr_vec(expr.values); + Some(expr) + } + + fn visit_expr_if_exp(&mut self, mut expr: ExprIfExp) -> Option { + self.generic_visit_expr_if_exp(expr) + } + + fn generic_visit_expr_if_exp(&mut self, mut expr: ExprIfExp) -> Option { + expr.test = Box::new( + self.visit_expr(*expr.test) + .expect("Cannot remove test from if expression"), + ); + expr.body = Box::new( + self.visit_expr(*expr.body) + .expect("Cannot remove body from if expression"), + ); + expr.orelse = Box::new( + self.visit_expr(*expr.orelse) + .expect("Cannot remove orelse from if expression"), + ); + Some(expr) + } + + fn visit_expr_lambda(&mut self, mut expr: ExprLambda) -> Option { + self.generic_visit_expr_lambda(expr) + } + + fn generic_visit_expr_lambda(&mut self, mut expr: ExprLambda) -> Option { + expr.args = Box::new(self.visit_arguments(*expr.args)); + expr.body = Box::new( + self.visit_expr(*expr.body) + .expect("Cannot remove body from lambda expression"), + ); + Some(expr) + } + + fn visit_expr_unary_op(&mut self, mut expr: ExprUnaryOp) -> Option { + self.generic_visit_expr_unary_op(expr) + } + + fn generic_visit_expr_unary_op(&mut self, mut expr: ExprUnaryOp) -> Option { + expr.operand = Box::new( + self.visit_expr(*expr.operand) + .expect("Cannot remove operand from unary operation"), + ); + Some(expr) + } + + fn visit_expr_bin_op(&mut self, mut expr: ExprBinOp) -> Option { + self.generic_visit_expr_bin_op(expr) + } + + fn generic_visit_expr_bin_op(&mut self, mut expr: ExprBinOp) -> Option { + expr.left = Box::new( + self.visit_expr(*expr.left) + .expect("Cannot remove left from binary operation"), + ); + expr.right = Box::new( + self.visit_expr(*expr.right) + .expect("Cannot remove right from binary operation"), + ); + Some(expr) + } + + fn visit_expr_named_expr(&mut self, mut expr: ExprNamedExpr) -> Option { + self.generic_visit_expr_named_expr(expr) + } + + fn generic_visit_expr_named_expr(&mut self, mut expr: ExprNamedExpr) -> Option { + expr.target = Box::new( + self.visit_expr(*expr.target) + .expect("Cannot remove target from named expression"), + ); + expr.value = Box::new( + self.visit_expr(*expr.value) + .expect("Cannot remove value from named expression"), + ); + Some(expr) + } + + fn visit_expr_bool_op(&mut self, mut expr: ExprBoolOp) -> Option { + self.generic_visit_expr_bool_op(expr) + } + + fn generic_visit_expr_bool_op(&mut self, mut expr: ExprBoolOp) -> Option { + expr.values = self.visit_expr_vec(expr.values); + if expr.values.len() == 0 { + panic!("Cannot remove all values from bool op"); + } + Some(expr) + } + + fn visit_arg(&mut self, arg: Arg) -> Option { + self.generic_visit_arg(arg) + } + + fn generic_visit_arg(&mut self, mut arg: Arg) -> Option { + if let Some(annotation) = arg.annotation { + arg.annotation = box_expr_option(self.visit_expr(*annotation)); + } + return Some(arg); + } + + fn visit_arg_with_default(&mut self, mut arg: ArgWithDefault) -> Option { + self.generic_visit_arg_with_default(arg) + } + + fn generic_visit_arg_with_default( + &mut self, + mut arg: ArgWithDefault, + ) -> Option { + arg.def = self + .visit_arg(arg.def) + .expect("Cannot remove def from arg with default"); + if let Some(default) = arg.default { + arg.default = box_expr_option(self.visit_expr(*default)); + } + + Some(arg) + } + + fn generic_visit_args_with_default_vec( + &mut self, + mut node: Vec, + ) -> Vec { + let mut new_nodes: Vec = Vec::new(); + + for arg in node { + if let Some(new_arg) = self.visit_arg_with_default(arg) { + new_nodes.push(new_arg); + } + } + return new_nodes; + } + + fn visit_arguments(&mut self, mut arguments: Arguments) -> Arguments { + self.generic_visit_arguments(arguments) + } + + fn generic_visit_arguments(&mut self, mut arguments: Arguments) -> Arguments { + arguments.args = self.generic_visit_args_with_default_vec(arguments.args); + if let Some(kwarg) = arguments.kwarg { + arguments.kwarg = self.visit_arg(*kwarg).map(|new_arg| Box::new(new_arg)); + } + arguments.kwonlyargs = self.generic_visit_args_with_default_vec(arguments.kwonlyargs); + arguments.posonlyargs = self.generic_visit_args_with_default_vec(arguments.posonlyargs); + if let Some(vararg) = arguments.vararg { + arguments.vararg = self.visit_arg(*vararg).map(|new_arg| Box::new(new_arg)); + } + return arguments; + } + + fn generic_visit_type_param_vec(&mut self, mut params: Vec) -> Vec { + let mut new_params: Vec = Vec::new(); + for param in params { + if let Some(new_param) = self.visit_type_param(param) { + new_params.push(new_param); + } + } + return new_params; + } + + fn visit_type_param(&mut self, mut param: TypeParam) -> Option { + self.generic_visit_type_param(param) + } + + fn generic_visit_type_param(&mut self, mut param: TypeParam) -> Option { + match param { + TypeParam::ParamSpec(param_spec) => self + .visit_type_param_spec(param_spec) + .map(|new_param| TypeParam::ParamSpec(new_param)), + TypeParam::TypeVar(param_var) => self + .visit_type_param_var(param_var) + .map(|new_param| TypeParam::TypeVar(new_param)), + TypeParam::TypeVarTuple(param_var_tuple) => self + .visit_type_param_var_tuple(param_var_tuple) + .map(|new_param| TypeParam::TypeVarTuple(new_param)), + } + } + + fn visit_type_param_spec( + &mut self, + mut param_spec: TypeParamParamSpec, + ) -> Option { + self.generic_visit_type_param_spec(param_spec) + } + + fn generic_visit_type_param_spec( + &mut self, + mut param_spec: TypeParamParamSpec, + ) -> Option { + Some(param_spec) + } + + fn visit_type_param_var( + &mut self, + mut param_var: TypeParamTypeVar, + ) -> Option { + self.generic_visit_type_param_var(param_var) + } + + fn generic_visit_type_param_var( + &mut self, + mut param_var: TypeParamTypeVar, + ) -> Option { + if let Some(bound) = param_var.bound { + param_var.bound = box_expr_option(self.visit_expr(*bound)); + } + Some(param_var) + } + + fn visit_type_param_var_tuple( + &mut self, + mut param_var_tuple: TypeParamTypeVarTuple, + ) -> Option { + self.generic_visit_type_param_var_tuple(param_var_tuple) + } + + fn generic_visit_type_param_var_tuple( + &mut self, + mut param_var_tuple: TypeParamTypeVarTuple, + ) -> Option { + Some(param_var_tuple) + } + + fn generic_visit_with_item_vec(&mut self, with_items: Vec) -> Vec { + let mut new_with_items: Vec = Vec::new(); + + for with_item in with_items { + if let Some(new_with_item) = self.visit_with_item(with_item) { + new_with_items.push(new_with_item); + } + } + + return new_with_items; + } + + fn visit_with_item(&mut self, mut with_item: WithItem) -> Option { + self.generic_visit_with_item(with_item) + } + + fn generic_visit_with_item(&mut self, mut with_item: WithItem) -> Option { + with_item.context_expr = self + .visit_expr(with_item.context_expr) + .expect("Cannot remove context expr from with item"); + if let Some(optional_vars) = with_item.optional_vars { + with_item.optional_vars = box_expr_option(self.visit_expr(*optional_vars)); + } + Some(with_item) + } +} From 34c960be4d90ff5038c92919bf801e125839abdc Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Fri, 24 Jan 2025 15:06:24 +0100 Subject: [PATCH 08/21] chore: bump version to 0.2.0 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a323c4d..496ffae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -484,7 +484,7 @@ dependencies = [ [[package]] name = "rustpython-unparser" -version = "0.1.4" +version = "0.2.0" dependencies = [ "pretty_assertions", "rand", diff --git a/Cargo.toml b/Cargo.toml index 023f6fc..61461c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustpython-unparser" -version = "0.1.4" +version = "0.2.0" edition = "2021" authors = ["Jan Vollmer "] readme = "README.md" From 341765bacccb3604f0f46fc9f880ec20f9cdf6a6 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Fri, 24 Jan 2025 15:15:12 +0100 Subject: [PATCH 09/21] fix: enable transformer feature in vscode rust-analyzer settings --- .vscode/settings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 1c4e1f7..2b65b73 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,5 +5,6 @@ "editor.defaultFormatter": "charliermarsh.ruff" }, "python.defaultInterpreterPath": ".venv/bin/python", - "python.terminal.activateEnvInCurrentTerminal": true + "python.terminal.activateEnvInCurrentTerminal": true, + "rust-analyzer.cargo.features": ["transformer"] } From 5245c002dbc3460fc86c09b0ebbc050b6dc967e3 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Sat, 8 Feb 2025 21:58:21 +0100 Subject: [PATCH 10/21] feat(transformer): add annotation enter and exit hooks (#12) --- src/transformer.rs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/transformer.rs b/src/transformer.rs index ea33cd2..ebd0901 100644 --- a/src/transformer.rs +++ b/src/transformer.rs @@ -19,6 +19,19 @@ fn box_expr_option(expr: Option) -> Option> { } #[allow(unused_mut)] pub trait Transformer { + #[allow(unused_variables)] + fn on_enter_annotation(&mut self, expr: &Expr) {} + #[allow(unused_variables)] + fn on_exit_annotation(&mut self, expr: &Option) {} + + fn visit_annotation(&mut self, expr: Box) -> Option { + let unboxed_annotation = *expr; + self.on_enter_annotation(&unboxed_annotation); + let new_annotation = self.visit_expr(unboxed_annotation); + self.on_exit_annotation(&new_annotation); + new_annotation + } + fn visit_stmt_vec(&mut self, stmts: Vec) -> Vec { let mut new_stmts: Vec = Vec::new(); @@ -741,7 +754,7 @@ pub trait Transformer { stmt.decorator_list = self.visit_expr_vec(stmt.decorator_list); stmt.args = Box::new(self.visit_arguments(*stmt.args)); if let Some(returns) = stmt.returns { - stmt.returns = box_expr_option(self.visit_expr(*returns)); + stmt.returns = box_expr_option(self.visit_annotation(returns)); } stmt.body = self.visit_stmt_vec(stmt.body); if stmt.body.len() == 0 { @@ -765,7 +778,7 @@ pub trait Transformer { stmt.decorator_list = self.visit_expr_vec(stmt.decorator_list); stmt.args = Box::new(self.visit_arguments(*stmt.args)); if let Some(returns) = stmt.returns { - stmt.returns = box_expr_option(self.visit_expr(*returns)); + stmt.returns = box_expr_option(self.visit_annotation(returns)); } stmt.body = self.visit_stmt_vec(stmt.body); if stmt.body.len() == 0 { @@ -822,7 +835,7 @@ pub trait Transformer { fn generic_visit_ann_assign(&mut self, mut stmt: StmtAnnAssign) -> Option { stmt.annotation = Box::new( - self.visit_expr(*stmt.annotation) + self.visit_annotation(stmt.annotation) .expect("Cannot remove annotation from annotated assignment"), ); @@ -1368,7 +1381,7 @@ pub trait Transformer { fn generic_visit_arg(&mut self, mut arg: Arg) -> Option { if let Some(annotation) = arg.annotation { - arg.annotation = box_expr_option(self.visit_expr(*annotation)); + arg.annotation = box_expr_option(self.visit_annotation(annotation)); } return Some(arg); } @@ -1476,7 +1489,7 @@ pub trait Transformer { mut param_var: TypeParamTypeVar, ) -> Option { if let Some(bound) = param_var.bound { - param_var.bound = box_expr_option(self.visit_expr(*bound)); + param_var.bound = box_expr_option(self.visit_annotation(bound)); } Some(param_var) } From 0e9ab0b8b750eceea7aae2e8fea39db61f2fb705 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 8 Feb 2025 21:58:48 +0100 Subject: [PATCH 11/21] [pre-commit.ci] pre-commit autoupdate (#11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.9.1 → v0.9.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.9.1...v0.9.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Jan Vollmer --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b579675..2b83677 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: hooks: - id: prettier - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.9.1" + rev: "v0.9.4" hooks: - id: ruff args: From fe1f2a155b1ca7a23b07fe5b517c927d5d699f20 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Sat, 8 Feb 2025 21:59:27 +0100 Subject: [PATCH 12/21] chore: bump version to 0.2.1 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 496ffae..53e06b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -484,7 +484,7 @@ dependencies = [ [[package]] name = "rustpython-unparser" -version = "0.2.0" +version = "0.2.1" dependencies = [ "pretty_assertions", "rand", diff --git a/Cargo.toml b/Cargo.toml index 61461c0..aeb6d90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustpython-unparser" -version = "0.2.0" +version = "0.2.1" edition = "2021" authors = ["Jan Vollmer "] readme = "README.md" From 4e9dc21b3c01b121246137cbf941aa1cf4b6a140 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Feb 2025 09:23:38 +0100 Subject: [PATCH 13/21] [pre-commit.ci] pre-commit autoupdate (#13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/jvllmr/pre-commit-prettier: v3.4.2 → v3.5.0](https://github.com/jvllmr/pre-commit-prettier/compare/v3.4.2...v3.5.0) - [github.com/astral-sh/ruff-pre-commit: v0.9.4 → v0.9.6](https://github.com/astral-sh/ruff-pre-commit/compare/v0.9.4...v0.9.6) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2b83677..5729358 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,11 +9,11 @@ repos: - id: cargo-fix - id: cargo-fmt - repo: https://github.com/jvllmr/pre-commit-prettier - rev: v3.4.2 + rev: v3.5.0 hooks: - id: prettier - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.9.4" + rev: "v0.9.6" hooks: - id: ruff args: From fe21b9d36a7819c6704b538ff2afd0670007061d Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Wed, 19 Feb 2025 19:57:28 +0100 Subject: [PATCH 14/21] fix(unparser): eagerly unparse float constants to scientific notation (#15) * fix(unparser): eagerly unparse float constants to scientific notation * fix build --- Cargo.toml | 2 +- src/unparser.rs | 34 ++++++++++++++++++++++++++++++++-- test_files/constants.py | 14 ++++++++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index aeb6d90..c58f938 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ rust-version = "1.72.1" transformer = [] [dependencies] -rustpython-ast = { version = ">=0.4.0" } +rustpython-ast = { version = ">=0.4.0", features = ["rustpython-literal"] } rustpython-literal = ">=0.4.0" [dev-dependencies] diff --git a/src/unparser.rs b/src/unparser.rs index 8b6368f..c268b0f 100644 --- a/src/unparser.rs +++ b/src/unparser.rs @@ -1057,6 +1057,7 @@ impl Unparser { } fn _unparse_constant(&mut self, constant: &Constant) { + let inf_str = "1e309"; return match constant { Constant::Tuple(values) => { self.write_str("("); @@ -1094,8 +1095,37 @@ impl Unparser { self.write_str(&escaped); } Constant::None => self.write_str("None"), - Constant::Complex { real, imag: _ } => self.write_str(&real.to_string()), - Constant::Float(value) => self.write_str(&value.to_string()), + Constant::Complex { real, imag } => { + if real.is_infinite() || imag.is_infinite() { + self.write_str(&constant.to_string().replace("inf", &inf_str)); + } else { + self.write_str(&constant.to_string()); + } + } + Constant::Float(value) => { + if value.is_infinite() { + self.write_str(inf_str); + } else { + let mut str_value = value.to_string(); + if value.fract() == 0.0 { + let mut trailing_zeroes = 0; + while str_value.ends_with("0") { + str_value.pop(); + trailing_zeroes += 1; + } + str_value = format!("{}e{}", str_value, trailing_zeroes); + } else if str_value.starts_with(&format!("0.{}", "0".repeat(5))) { + let mut trimmed = str_value[2..].to_owned(); + let mut factor = 1; + while trimmed.starts_with("0") { + trimmed = trimmed[1..].to_owned(); + factor += 1; + } + str_value = format!("{}e-{}", trimmed, factor); + } + self.write_str(&str_value); + } + } }; } diff --git a/test_files/constants.py b/test_files/constants.py index e0f67f4..95ef7c2 100644 --- a/test_files/constants.py +++ b/test_files/constants.py @@ -1,3 +1,17 @@ escaped_string_expr = "'\"'''\"\"\"{}\\" escaped_unicode_expr = u"'\"'''\"\"\"{}\\" # fmt: skip escaped_bytes_expr = b"'\"'''\"\"\"{}\\" + + +# real code from yaml.constructor +# https://github.com/yaml/pyyaml/blob/69c141adcf805c5ebdc9ba519927642ee5c7f639/lib/yaml/constructor.py#L265 +inf_value = 1e300 + +# if inf_value would not be initialized in scientific notation +# the following loop would run for a long time +while inf_value != inf_value * inf_value: + inf_value *= inf_value + +scientific_fraction = 1e-69 + +complex_number = 3 + 4j From f20f578c5ec87d27edbe49b9efdef76c105b8001 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Wed, 19 Feb 2025 20:00:25 +0100 Subject: [PATCH 15/21] chore: bump version to 0.2.2 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 53e06b7..e9244c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -484,7 +484,7 @@ dependencies = [ [[package]] name = "rustpython-unparser" -version = "0.2.1" +version = "0.2.2" dependencies = [ "pretty_assertions", "rand", diff --git a/Cargo.toml b/Cargo.toml index c58f938..7713931 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustpython-unparser" -version = "0.2.1" +version = "0.2.2" edition = "2021" authors = ["Jan Vollmer "] readme = "README.md" From c5301b2f83114ab98ba77822ebabb2653fea82dd Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Wed, 19 Feb 2025 20:06:46 +0100 Subject: [PATCH 16/21] feat(ci): enable workflow_dispatch on release workflow --- .github/workflows/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 970c30f..532a9b9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,6 +3,7 @@ name: Upload to crates.io on: release: types: [published] + workflow_dispatch: env: CARGO_TERM_COLOR: always From 0cf4e19d46e20a8d8fd3323052777f831ed97803 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Wed, 19 Feb 2025 20:32:01 +0100 Subject: [PATCH 17/21] fix(unparser): fixed precedence for subscriptions (#16) --- src/unparser.rs | 4 +++- test_files/precedence.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 test_files/precedence.py diff --git a/src/unparser.rs b/src/unparser.rs index c268b0f..7882ecd 100644 --- a/src/unparser.rs +++ b/src/unparser.rs @@ -1142,7 +1142,9 @@ impl Unparser { self.write_str(&node.attr); } fn unparse_expr_subscript(&mut self, node: &ExprSubscript) { - self.unparse_expr(&node.value); + self.with_precedence(Precedence::Atom, |prec_self| { + prec_self.unparse_expr(&node.value); + }); self.write_str("["); self.unparse_expr(&node.slice); self.write_str("]"); diff --git a/test_files/precedence.py b/test_files/precedence.py new file mode 100644 index 0000000..2432dca --- /dev/null +++ b/test_files/precedence.py @@ -0,0 +1,2 @@ +reduce = "abcdefhijklmnopqrstuvwxyz" +reduce_list = (list(reduce) + [None] * 5)[:5] From 4abdfa55095abb9ea6af747e4349959a054f1f54 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Wed, 19 Feb 2025 20:32:59 +0100 Subject: [PATCH 18/21] chore: bump version to 0.2.3 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e9244c3..792a386 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -484,7 +484,7 @@ dependencies = [ [[package]] name = "rustpython-unparser" -version = "0.2.2" +version = "0.2.3" dependencies = [ "pretty_assertions", "rand", diff --git a/Cargo.toml b/Cargo.toml index 7713931..8c6e9e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustpython-unparser" -version = "0.2.2" +version = "0.2.3" edition = "2021" authors = ["Jan Vollmer "] readme = "README.md" From 7898e37015f793ba1380a8c9fcf25c6f78348ee9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 8 Mar 2025 18:15:35 +0100 Subject: [PATCH 19/21] [pre-commit.ci] pre-commit autoupdate (#14) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/jvllmr/pre-commit-prettier: v3.5.0 → v3.5.3](https://github.com/jvllmr/pre-commit-prettier/compare/v3.5.0...v3.5.3) - [github.com/astral-sh/ruff-pre-commit: v0.9.6 → v0.9.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.9.6...v0.9.9) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5729358..82e5f4f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,11 +9,11 @@ repos: - id: cargo-fix - id: cargo-fmt - repo: https://github.com/jvllmr/pre-commit-prettier - rev: v3.5.0 + rev: v3.5.3 hooks: - id: prettier - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.9.6" + rev: "v0.9.9" hooks: - id: ruff args: From 3e9f4ee877f585904c1ba02da27667672747696f Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Sat, 8 Mar 2025 18:16:42 +0100 Subject: [PATCH 20/21] fix(unparser): correct precedences for comprehensions and generators (#17) --- src/unparser.rs | 48 ++++++++++++++++++++++++++++------------ test_files/precedence.py | 18 +++++++++++++++ 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/src/unparser.rs b/src/unparser.rs index 7882ecd..b57681c 100644 --- a/src/unparser.rs +++ b/src/unparser.rs @@ -789,11 +789,15 @@ impl Unparser { fn unparse_expr_if_exp(&mut self, node: &ExprIfExp) { let enum_member = Expr::IfExp(node.to_owned()); self.delimit_precedence(&enum_member, |block_self| { - block_self.unparse_expr(&node.body); - block_self.write_str(" if "); - block_self.unparse_expr(&node.test); - block_self.write_str(" else "); - block_self.unparse_expr(&node.orelse); + block_self.with_precedence_num(Precedence::Test.value() + 1, |prec_self| { + prec_self.unparse_expr(&node.body); + prec_self.write_str(" if "); + prec_self.unparse_expr(&node.test); + }); + block_self.with_precedence(Precedence::Test, |prec_self| { + prec_self.write_str(" else "); + prec_self.unparse_expr(&node.orelse); + }); }) } @@ -811,7 +815,10 @@ impl Unparser { self.write_str("**"); } } - self.unparse_expr(value); + self.with_precedence_num(EXPR_PRECEDENCE, |prec_self| { + prec_self.unparse_expr(value); + }); + if zipped.peek().is_some() { self.write_str(", "); } @@ -1173,14 +1180,21 @@ impl Unparser { fn unparse_expr_tuple(&mut self, node: &ExprTuple) { let mut elts_iter = node.elts.iter().peekable(); - self.write_str("("); + let should_delimit = + node.elts.len() == 0 || self.precedence_level > Precedence::Tuple.value(); + if should_delimit { + self.write_str("("); + } + while let Some(expr) = elts_iter.next() { self.unparse_expr(expr); if elts_iter.peek().is_some() || node.elts.len() == 1 { self.write_str(", "); } } - self.write_str(")"); + if should_delimit { + self.write_str(")"); + } } fn unparse_expr_slice(&mut self, node: &ExprSlice) { @@ -1221,13 +1235,19 @@ impl Unparser { } else { self.write_str(" for "); } - self.unparse_expr(&node.target); + self.with_precedence(Precedence::Tuple, |prec_self| { + prec_self.unparse_expr(&node.target); + }); + self.write_str(" in "); - self.unparse_expr(&node.iter); - for if_ in &node.ifs { - self.write_str(" if "); - self.unparse_expr(if_); - } + + self.with_precedence_num(Precedence::Test.value() + 1, |prec_self| { + prec_self.unparse_expr(&node.iter); + for if_ in &node.ifs { + prec_self.write_str(" if "); + prec_self.unparse_expr(if_); + } + }); } fn unparse_excepthandler(&mut self, node: &ExceptHandler) { diff --git a/test_files/precedence.py b/test_files/precedence.py index 2432dca..b1ad95a 100644 --- a/test_files/precedence.py +++ b/test_files/precedence.py @@ -1,2 +1,20 @@ +import dataclasses + reduce = "abcdefhijklmnopqrstuvwxyz" reduce_list = (list(reduce) + [None] * 5)[:5] + + +@dataclasses.dataclass +class MyDataclass: + name: str + + +# from pydantic._internal._fields +dataclass_fields = { + field.name + for field in ( + dataclasses.fields(MyDataclass) if dataclasses.is_dataclass(MyDataclass) else () + ) +} + +a, b, c = "a", "b", "c" From 2b6fb51e8a85da52c3cc697157445fb9882e4162 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Sat, 8 Mar 2025 18:17:18 +0100 Subject: [PATCH 21/21] chore: bump version to 0.2.4 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 792a386..72f5b51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -484,7 +484,7 @@ dependencies = [ [[package]] name = "rustpython-unparser" -version = "0.2.3" +version = "0.2.4" dependencies = [ "pretty_assertions", "rand", diff --git a/Cargo.toml b/Cargo.toml index 8c6e9e0..13d986f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustpython-unparser" -version = "0.2.3" +version = "0.2.4" edition = "2021" authors = ["Jan Vollmer "] readme = "README.md"