From 5245c002dbc3460fc86c09b0ebbc050b6dc967e3 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Sat, 8 Feb 2025 21:58:21 +0100 Subject: [PATCH 01/12] feat(transformer): add annotation enter and exit hooks (#12) --- src/transformer.rs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/transformer.rs b/src/transformer.rs index ea33cd2..ebd0901 100644 --- a/src/transformer.rs +++ b/src/transformer.rs @@ -19,6 +19,19 @@ fn box_expr_option(expr: Option) -> Option> { } #[allow(unused_mut)] pub trait Transformer { + #[allow(unused_variables)] + fn on_enter_annotation(&mut self, expr: &Expr) {} + #[allow(unused_variables)] + fn on_exit_annotation(&mut self, expr: &Option) {} + + fn visit_annotation(&mut self, expr: Box) -> Option { + let unboxed_annotation = *expr; + self.on_enter_annotation(&unboxed_annotation); + let new_annotation = self.visit_expr(unboxed_annotation); + self.on_exit_annotation(&new_annotation); + new_annotation + } + fn visit_stmt_vec(&mut self, stmts: Vec) -> Vec { let mut new_stmts: Vec = Vec::new(); @@ -741,7 +754,7 @@ pub trait Transformer { stmt.decorator_list = self.visit_expr_vec(stmt.decorator_list); stmt.args = Box::new(self.visit_arguments(*stmt.args)); if let Some(returns) = stmt.returns { - stmt.returns = box_expr_option(self.visit_expr(*returns)); + stmt.returns = box_expr_option(self.visit_annotation(returns)); } stmt.body = self.visit_stmt_vec(stmt.body); if stmt.body.len() == 0 { @@ -765,7 +778,7 @@ pub trait Transformer { stmt.decorator_list = self.visit_expr_vec(stmt.decorator_list); stmt.args = Box::new(self.visit_arguments(*stmt.args)); if let Some(returns) = stmt.returns { - stmt.returns = box_expr_option(self.visit_expr(*returns)); + stmt.returns = box_expr_option(self.visit_annotation(returns)); } stmt.body = self.visit_stmt_vec(stmt.body); if stmt.body.len() == 0 { @@ -822,7 +835,7 @@ pub trait Transformer { fn generic_visit_ann_assign(&mut self, mut stmt: StmtAnnAssign) -> Option { stmt.annotation = Box::new( - self.visit_expr(*stmt.annotation) + self.visit_annotation(stmt.annotation) .expect("Cannot remove annotation from annotated assignment"), ); @@ -1368,7 +1381,7 @@ pub trait Transformer { fn generic_visit_arg(&mut self, mut arg: Arg) -> Option { if let Some(annotation) = arg.annotation { - arg.annotation = box_expr_option(self.visit_expr(*annotation)); + arg.annotation = box_expr_option(self.visit_annotation(annotation)); } return Some(arg); } @@ -1476,7 +1489,7 @@ pub trait Transformer { mut param_var: TypeParamTypeVar, ) -> Option { if let Some(bound) = param_var.bound { - param_var.bound = box_expr_option(self.visit_expr(*bound)); + param_var.bound = box_expr_option(self.visit_annotation(bound)); } Some(param_var) } From 0e9ab0b8b750eceea7aae2e8fea39db61f2fb705 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 8 Feb 2025 21:58:48 +0100 Subject: [PATCH 02/12] [pre-commit.ci] pre-commit autoupdate (#11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.9.1 → v0.9.4](https://github.com/astral-sh/ruff-pre-commit/compare/v0.9.1...v0.9.4) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Jan Vollmer --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b579675..2b83677 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: hooks: - id: prettier - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.9.1" + rev: "v0.9.4" hooks: - id: ruff args: From fe1f2a155b1ca7a23b07fe5b517c927d5d699f20 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Sat, 8 Feb 2025 21:59:27 +0100 Subject: [PATCH 03/12] chore: bump version to 0.2.1 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 496ffae..53e06b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -484,7 +484,7 @@ dependencies = [ [[package]] name = "rustpython-unparser" -version = "0.2.0" +version = "0.2.1" dependencies = [ "pretty_assertions", "rand", diff --git a/Cargo.toml b/Cargo.toml index 61461c0..aeb6d90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustpython-unparser" -version = "0.2.0" +version = "0.2.1" edition = "2021" authors = ["Jan Vollmer "] readme = "README.md" From 4e9dc21b3c01b121246137cbf941aa1cf4b6a140 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Feb 2025 09:23:38 +0100 Subject: [PATCH 04/12] [pre-commit.ci] pre-commit autoupdate (#13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/jvllmr/pre-commit-prettier: v3.4.2 → v3.5.0](https://github.com/jvllmr/pre-commit-prettier/compare/v3.4.2...v3.5.0) - [github.com/astral-sh/ruff-pre-commit: v0.9.4 → v0.9.6](https://github.com/astral-sh/ruff-pre-commit/compare/v0.9.4...v0.9.6) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2b83677..5729358 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,11 +9,11 @@ repos: - id: cargo-fix - id: cargo-fmt - repo: https://github.com/jvllmr/pre-commit-prettier - rev: v3.4.2 + rev: v3.5.0 hooks: - id: prettier - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.9.4" + rev: "v0.9.6" hooks: - id: ruff args: From fe21b9d36a7819c6704b538ff2afd0670007061d Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Wed, 19 Feb 2025 19:57:28 +0100 Subject: [PATCH 05/12] fix(unparser): eagerly unparse float constants to scientific notation (#15) * fix(unparser): eagerly unparse float constants to scientific notation * fix build --- Cargo.toml | 2 +- src/unparser.rs | 34 ++++++++++++++++++++++++++++++++-- test_files/constants.py | 14 ++++++++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index aeb6d90..c58f938 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ rust-version = "1.72.1" transformer = [] [dependencies] -rustpython-ast = { version = ">=0.4.0" } +rustpython-ast = { version = ">=0.4.0", features = ["rustpython-literal"] } rustpython-literal = ">=0.4.0" [dev-dependencies] diff --git a/src/unparser.rs b/src/unparser.rs index 8b6368f..c268b0f 100644 --- a/src/unparser.rs +++ b/src/unparser.rs @@ -1057,6 +1057,7 @@ impl Unparser { } fn _unparse_constant(&mut self, constant: &Constant) { + let inf_str = "1e309"; return match constant { Constant::Tuple(values) => { self.write_str("("); @@ -1094,8 +1095,37 @@ impl Unparser { self.write_str(&escaped); } Constant::None => self.write_str("None"), - Constant::Complex { real, imag: _ } => self.write_str(&real.to_string()), - Constant::Float(value) => self.write_str(&value.to_string()), + Constant::Complex { real, imag } => { + if real.is_infinite() || imag.is_infinite() { + self.write_str(&constant.to_string().replace("inf", &inf_str)); + } else { + self.write_str(&constant.to_string()); + } + } + Constant::Float(value) => { + if value.is_infinite() { + self.write_str(inf_str); + } else { + let mut str_value = value.to_string(); + if value.fract() == 0.0 { + let mut trailing_zeroes = 0; + while str_value.ends_with("0") { + str_value.pop(); + trailing_zeroes += 1; + } + str_value = format!("{}e{}", str_value, trailing_zeroes); + } else if str_value.starts_with(&format!("0.{}", "0".repeat(5))) { + let mut trimmed = str_value[2..].to_owned(); + let mut factor = 1; + while trimmed.starts_with("0") { + trimmed = trimmed[1..].to_owned(); + factor += 1; + } + str_value = format!("{}e-{}", trimmed, factor); + } + self.write_str(&str_value); + } + } }; } diff --git a/test_files/constants.py b/test_files/constants.py index e0f67f4..95ef7c2 100644 --- a/test_files/constants.py +++ b/test_files/constants.py @@ -1,3 +1,17 @@ escaped_string_expr = "'\"'''\"\"\"{}\\" escaped_unicode_expr = u"'\"'''\"\"\"{}\\" # fmt: skip escaped_bytes_expr = b"'\"'''\"\"\"{}\\" + + +# real code from yaml.constructor +# https://github.com/yaml/pyyaml/blob/69c141adcf805c5ebdc9ba519927642ee5c7f639/lib/yaml/constructor.py#L265 +inf_value = 1e300 + +# if inf_value would not be initialized in scientific notation +# the following loop would run for a long time +while inf_value != inf_value * inf_value: + inf_value *= inf_value + +scientific_fraction = 1e-69 + +complex_number = 3 + 4j From f20f578c5ec87d27edbe49b9efdef76c105b8001 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Wed, 19 Feb 2025 20:00:25 +0100 Subject: [PATCH 06/12] chore: bump version to 0.2.2 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 53e06b7..e9244c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -484,7 +484,7 @@ dependencies = [ [[package]] name = "rustpython-unparser" -version = "0.2.1" +version = "0.2.2" dependencies = [ "pretty_assertions", "rand", diff --git a/Cargo.toml b/Cargo.toml index c58f938..7713931 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustpython-unparser" -version = "0.2.1" +version = "0.2.2" edition = "2021" authors = ["Jan Vollmer "] readme = "README.md" From c5301b2f83114ab98ba77822ebabb2653fea82dd Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Wed, 19 Feb 2025 20:06:46 +0100 Subject: [PATCH 07/12] feat(ci): enable workflow_dispatch on release workflow --- .github/workflows/publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 970c30f..532a9b9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -3,6 +3,7 @@ name: Upload to crates.io on: release: types: [published] + workflow_dispatch: env: CARGO_TERM_COLOR: always From 0cf4e19d46e20a8d8fd3323052777f831ed97803 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Wed, 19 Feb 2025 20:32:01 +0100 Subject: [PATCH 08/12] fix(unparser): fixed precedence for subscriptions (#16) --- src/unparser.rs | 4 +++- test_files/precedence.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 test_files/precedence.py diff --git a/src/unparser.rs b/src/unparser.rs index c268b0f..7882ecd 100644 --- a/src/unparser.rs +++ b/src/unparser.rs @@ -1142,7 +1142,9 @@ impl Unparser { self.write_str(&node.attr); } fn unparse_expr_subscript(&mut self, node: &ExprSubscript) { - self.unparse_expr(&node.value); + self.with_precedence(Precedence::Atom, |prec_self| { + prec_self.unparse_expr(&node.value); + }); self.write_str("["); self.unparse_expr(&node.slice); self.write_str("]"); diff --git a/test_files/precedence.py b/test_files/precedence.py new file mode 100644 index 0000000..2432dca --- /dev/null +++ b/test_files/precedence.py @@ -0,0 +1,2 @@ +reduce = "abcdefhijklmnopqrstuvwxyz" +reduce_list = (list(reduce) + [None] * 5)[:5] From 4abdfa55095abb9ea6af747e4349959a054f1f54 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Wed, 19 Feb 2025 20:32:59 +0100 Subject: [PATCH 09/12] chore: bump version to 0.2.3 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e9244c3..792a386 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -484,7 +484,7 @@ dependencies = [ [[package]] name = "rustpython-unparser" -version = "0.2.2" +version = "0.2.3" dependencies = [ "pretty_assertions", "rand", diff --git a/Cargo.toml b/Cargo.toml index 7713931..8c6e9e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustpython-unparser" -version = "0.2.2" +version = "0.2.3" edition = "2021" authors = ["Jan Vollmer "] readme = "README.md" From 7898e37015f793ba1380a8c9fcf25c6f78348ee9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 8 Mar 2025 18:15:35 +0100 Subject: [PATCH 10/12] [pre-commit.ci] pre-commit autoupdate (#14) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/jvllmr/pre-commit-prettier: v3.5.0 → v3.5.3](https://github.com/jvllmr/pre-commit-prettier/compare/v3.5.0...v3.5.3) - [github.com/astral-sh/ruff-pre-commit: v0.9.6 → v0.9.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.9.6...v0.9.9) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5729358..82e5f4f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,11 +9,11 @@ repos: - id: cargo-fix - id: cargo-fmt - repo: https://github.com/jvllmr/pre-commit-prettier - rev: v3.5.0 + rev: v3.5.3 hooks: - id: prettier - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.9.6" + rev: "v0.9.9" hooks: - id: ruff args: From 3e9f4ee877f585904c1ba02da27667672747696f Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Sat, 8 Mar 2025 18:16:42 +0100 Subject: [PATCH 11/12] fix(unparser): correct precedences for comprehensions and generators (#17) --- src/unparser.rs | 48 ++++++++++++++++++++++++++++------------ test_files/precedence.py | 18 +++++++++++++++ 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/src/unparser.rs b/src/unparser.rs index 7882ecd..b57681c 100644 --- a/src/unparser.rs +++ b/src/unparser.rs @@ -789,11 +789,15 @@ impl Unparser { fn unparse_expr_if_exp(&mut self, node: &ExprIfExp) { let enum_member = Expr::IfExp(node.to_owned()); self.delimit_precedence(&enum_member, |block_self| { - block_self.unparse_expr(&node.body); - block_self.write_str(" if "); - block_self.unparse_expr(&node.test); - block_self.write_str(" else "); - block_self.unparse_expr(&node.orelse); + block_self.with_precedence_num(Precedence::Test.value() + 1, |prec_self| { + prec_self.unparse_expr(&node.body); + prec_self.write_str(" if "); + prec_self.unparse_expr(&node.test); + }); + block_self.with_precedence(Precedence::Test, |prec_self| { + prec_self.write_str(" else "); + prec_self.unparse_expr(&node.orelse); + }); }) } @@ -811,7 +815,10 @@ impl Unparser { self.write_str("**"); } } - self.unparse_expr(value); + self.with_precedence_num(EXPR_PRECEDENCE, |prec_self| { + prec_self.unparse_expr(value); + }); + if zipped.peek().is_some() { self.write_str(", "); } @@ -1173,14 +1180,21 @@ impl Unparser { fn unparse_expr_tuple(&mut self, node: &ExprTuple) { let mut elts_iter = node.elts.iter().peekable(); - self.write_str("("); + let should_delimit = + node.elts.len() == 0 || self.precedence_level > Precedence::Tuple.value(); + if should_delimit { + self.write_str("("); + } + while let Some(expr) = elts_iter.next() { self.unparse_expr(expr); if elts_iter.peek().is_some() || node.elts.len() == 1 { self.write_str(", "); } } - self.write_str(")"); + if should_delimit { + self.write_str(")"); + } } fn unparse_expr_slice(&mut self, node: &ExprSlice) { @@ -1221,13 +1235,19 @@ impl Unparser { } else { self.write_str(" for "); } - self.unparse_expr(&node.target); + self.with_precedence(Precedence::Tuple, |prec_self| { + prec_self.unparse_expr(&node.target); + }); + self.write_str(" in "); - self.unparse_expr(&node.iter); - for if_ in &node.ifs { - self.write_str(" if "); - self.unparse_expr(if_); - } + + self.with_precedence_num(Precedence::Test.value() + 1, |prec_self| { + prec_self.unparse_expr(&node.iter); + for if_ in &node.ifs { + prec_self.write_str(" if "); + prec_self.unparse_expr(if_); + } + }); } fn unparse_excepthandler(&mut self, node: &ExceptHandler) { diff --git a/test_files/precedence.py b/test_files/precedence.py index 2432dca..b1ad95a 100644 --- a/test_files/precedence.py +++ b/test_files/precedence.py @@ -1,2 +1,20 @@ +import dataclasses + reduce = "abcdefhijklmnopqrstuvwxyz" reduce_list = (list(reduce) + [None] * 5)[:5] + + +@dataclasses.dataclass +class MyDataclass: + name: str + + +# from pydantic._internal._fields +dataclass_fields = { + field.name + for field in ( + dataclasses.fields(MyDataclass) if dataclasses.is_dataclass(MyDataclass) else () + ) +} + +a, b, c = "a", "b", "c" From 2b6fb51e8a85da52c3cc697157445fb9882e4162 Mon Sep 17 00:00:00 2001 From: Jan Vollmer Date: Sat, 8 Mar 2025 18:17:18 +0100 Subject: [PATCH 12/12] chore: bump version to 0.2.4 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 792a386..72f5b51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -484,7 +484,7 @@ dependencies = [ [[package]] name = "rustpython-unparser" -version = "0.2.3" +version = "0.2.4" dependencies = [ "pretty_assertions", "rand", diff --git a/Cargo.toml b/Cargo.toml index 8c6e9e0..13d986f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rustpython-unparser" -version = "0.2.3" +version = "0.2.4" edition = "2021" authors = ["Jan Vollmer "] readme = "README.md"