From 94a7b485602c5e578d7f593a6769ad7c55ec2612 Mon Sep 17 00:00:00 2001 From: TheAnyKey Date: Tue, 28 Apr 2020 18:41:17 -0400 Subject: [PATCH 01/13] added initial implementation of self defining expressions to fstring (Python3.8) --- parser/src/fstring.rs | 72 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 8 deletions(-) diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index 79a7524690..409fa77465 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -26,6 +26,7 @@ impl<'a> FStringParser<'a> { let mut spec = None; let mut delims = Vec::new(); let mut conversion = None; + let mut pred_expression_text = String::new(); while let Some(ch) = self.chars.next() { match ch { @@ -50,6 +51,13 @@ impl<'a> FStringParser<'a> { return Err(ExpectedRbrace); } } + + // match a python 3.8 self documenting expression + // format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}' + '=' if self.chars.peek() != Some(&'=') => { + pred_expression_text = expression.trim().to_string(); // use expression and remove whitespace + } + ':' if delims.is_empty() => { let mut nested = false; let mut in_nested = false; @@ -121,14 +129,34 @@ impl<'a> FStringParser<'a> { if expression.is_empty() { return Err(EmptyExpression); } - return Ok(FormattedValue { - value: Box::new( - parse_expression(expression.trim()) - .map_err(|e| InvalidExpression(Box::new(e.error)))?, - ), - conversion, - spec, - }); + if pred_expression_text.is_empty() { + return Ok(FormattedValue { + value: Box::new( + parse_expression(expression.trim()) + .map_err(|e| InvalidExpression(Box::new(e.error)))?, + ), + conversion, + spec, + }); + } + else { + return Ok(Joined{ + values:vec![ + Constant{ + value:pred_expression_text.to_owned() + }, + + FormattedValue { + value: Box::new( + parse_expression(expression.trim()) + .map_err(|e| InvalidExpression(Box::new(e.error)))?, + ), + conversion, + spec,}, + ] + } + ); + } } '"' | '\'' => { expression.push(ch); @@ -298,6 +326,30 @@ mod tests { ); } + #[test] + fn test_fstring_parse_selfdocumenting_base() { + let src=String::from("{user=}"); + let parse_ast=parse_fstring(&src); + + assert!(parse_ast.is_ok()); + } + + #[test] + fn test_fstring_parse_selfdocumenting_base_more() { + let src=String::from("mix {user=} with text and {second=}"); + let parse_ast=parse_fstring(&src); + + assert!(parse_ast.is_ok()); + } + + #[test] + fn test_fstring_parse_selfdocumenting_format() { + let src=String::from("{user=:>10}"); + let parse_ast=parse_fstring(&src); + + assert!(parse_ast.is_ok()); + } + #[test] fn test_parse_invalid_fstring() { assert_eq!(parse_fstring("{5!a"), Err(ExpectedRbrace)); @@ -318,6 +370,10 @@ mod tests { assert_eq!(parse_fstring("{a:{b}"), Err(UnclosedLbrace)); assert_eq!(parse_fstring("{"), Err(UnclosedLbrace)); + // TODO: which err? + //assert_eq!(parse_fstring("{}"), Err()); // which one? + assert!(parse_fstring("{}").is_err()); // take this for the moment + // TODO: check for InvalidExpression enum? assert!(parse_fstring("{class}").is_err()); } From 34478d3db7b6482ce21f3f537e71e89bfe01f482 Mon Sep 17 00:00:00 2001 From: TheAnyKey Date: Wed, 29 Apr 2020 15:37:47 -0400 Subject: [PATCH 02/13] implemented selfdocumenting fstrings and tests from cpython- current handling of quotes is too simple --- parser/src/fstring.rs | 68 +++++++++++++++++++++++++++++--- tests/snippets/fstrings.py | 81 +++++++++++++++++++++++++++++++++++++- 2 files changed, 142 insertions(+), 7 deletions(-) diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index 409fa77465..1bf6a465e5 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -27,9 +27,36 @@ impl<'a> FStringParser<'a> { let mut delims = Vec::new(); let mut conversion = None; let mut pred_expression_text = String::new(); + let mut trailing_seq=String::new(); while let Some(ch) = self.chars.next() { match ch { + // can be integrated better with the remainign code, but as a starting point ok + // in general I would do here a tokenizing of the fstrings to omit this peeking. + '!' if self.chars.peek() == Some(&'=') => { + expression.push('!'); + expression.push('='); + self.chars.next(); + } + + '=' if self.chars.peek() == Some(&'=') => { + expression.push('='); + expression.push('='); + self.chars.next(); + } + + '>' if self.chars.peek() == Some(&'=') => { + expression.push('>'); + expression.push('='); + self.chars.next(); + } + + '<' if self.chars.peek() == Some(&'=') => { + expression.push('<'); + expression.push('='); + self.chars.next(); + } + '!' if delims.is_empty() && self.chars.peek() != Some(&'=') => { if expression.trim().is_empty() { return Err(EmptyExpression); @@ -54,8 +81,9 @@ impl<'a> FStringParser<'a> { // match a python 3.8 self documenting expression // format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}' - '=' if self.chars.peek() != Some(&'=') => { - pred_expression_text = expression.trim().to_string(); // use expression and remove whitespace + '=' if self.chars.peek() != Some(&'=') => { // check for delims empty? + pred_expression_text = expression.to_string(); // safe expression before = to print it + // pred_expression_text contains trailing spaces, which are trimmed somewhere later before printing, this needs to be prevented orovercome here } ':' if delims.is_empty() => { @@ -143,7 +171,11 @@ impl<'a> FStringParser<'a> { return Ok(Joined{ values:vec![ Constant{ - value:pred_expression_text.to_owned() + value:pred_expression_text.to_owned()+"=" + }, + + Constant { + value:trailing_seq.to_owned() }, FormattedValue { @@ -167,6 +199,11 @@ impl<'a> FStringParser<'a> { } } } + + ' ' if !pred_expression_text.is_empty() => { + trailing_seq.push(ch); + } + _ => { expression.push(ch); } @@ -370,9 +407,7 @@ mod tests { assert_eq!(parse_fstring("{a:{b}"), Err(UnclosedLbrace)); assert_eq!(parse_fstring("{"), Err(UnclosedLbrace)); - // TODO: which err? - //assert_eq!(parse_fstring("{}"), Err()); // which one? - assert!(parse_fstring("{}").is_err()); // take this for the moment + assert_eq!(parse_fstring("{}"), Err(EmptyExpression)); // TODO: check for InvalidExpression enum? assert!(parse_fstring("{class}").is_err()); @@ -384,4 +419,25 @@ mod tests { let parse_ast = parse_fstring(&source); assert!(parse_ast.is_ok()); } + + #[test] + fn test_parse_fstring_equals() { + let source = String::from("{42 == 42}"); + let parse_ast = parse_fstring(&source); + assert!(parse_ast.is_ok()); + } + + #[test] + fn test_parse_fstring_selfdoc_prec_space() { + let source = String::from("{x =}"); + let parse_ast = parse_fstring(&source); + assert!(parse_ast.is_ok()); + } + + #[test] + fn test_parse_fstring_selfdoc_trailing_space() { + let source = String::from("{x= }"); + let parse_ast = parse_fstring(&source); + assert!(parse_ast.is_ok()); + } } diff --git a/tests/snippets/fstrings.py b/tests/snippets/fstrings.py index 2d25abf6dc..8b1e1d5cd0 100644 --- a/tests/snippets/fstrings.py +++ b/tests/snippets/fstrings.py @@ -1,4 +1,4 @@ -from testutils import assert_raises +#from testutils import assert_raises foo = 'bar' assert f"{''}" == '' @@ -17,6 +17,13 @@ assert f'{16:0>+#10x}' == '00000+0x10' assert f"{{{(lambda x: f'hello, {x}')('world}')}" == '{hello, world}' +assert f'{foo=}' == 'foo=bar' + +num=42 +assert f'{num=}' == 'num=42' +assert f'{num=:>10}' == 'num= 42' + + spec = "0>+#10x" assert f"{16:{spec}}{foo}" == '00000+0x10bar' @@ -61,3 +68,75 @@ def __str__(self): assert f'>{v!r}' == ">'\u262e'" assert f'>{v!s}' == '>\u262e' assert f'>{v!a}' == r">'\u262e'" + + + + + +### Tests for fstring selfdocumenting form CPython + +class C: + def assertEqual(self, a,b): + assert a==b, "{0} == {1}".format(a,b) + +self=C() + +x = 'A string' +self.assertEqual(f'{10=}', '10=10') +self.assertEqual(f'{x=}', 'x=' + x )#repr(x)) # TODO: add ' when printing strings +#self.assertEqual(f'{x =}', 'x =' + x )# + repr(x)) # TODO: remove trim in impl +# self.assertEqual(f'{x=!s}', 'x=' + str(x)) # TODO : implement format specs properly +# self.assertEqual(f'{x=!r}', 'x=' + x) #repr(x)) +#self.assertEqual(f'{x=!a}', 'x=' + ascii(x)) + +x = 2.71828 +self.assertEqual(f'{x=:.2f}', 'x=' + format(x, '.2f')) +self.assertEqual(f'{x=:}', 'x=' + format(x, '')) +# self.assertEqual(f'{x=!r:^20}', 'x=' + format(repr(x), '^20')) +# self.assertEqual(f'{x=!s:^20}', 'x=' + format(str(x), '^20')) +# self.assertEqual(f'{x=!a:^20}', 'x=' + format(ascii(x), '^20')) + +x = 9 +self.assertEqual(f'{3*x+15=}', '3*x+15=42') + +# There is code in ast.c that deals with non-ascii expression values. So, +# use a unicode identifier to trigger that. +tenπ = 31.4 +self.assertEqual(f'{tenπ=:.2f}', 'tenπ=31.40') + +# Also test with Unicode in non-identifiers. +#self.assertEqual(f'{"Σ"=}', '"Σ"=\'Σ\'') ' TODO ' missing + +# Make sure nested fstrings still work. +self.assertEqual(f'{f"{3.1415=:.1f}":*^20}', '*****3.1415=3.1*****') + +# Make sure text before and after an expression with = works +# correctly. +pi = 'π' +#self.assertEqual(f'alpha α {pi=} ω omega', "alpha α pi='π' ω omega") # ' missing around pi + +# Check multi-line expressions. +#self.assertEqual(f'''{3=}''', '\n3\n=3') # TODO: multiline f strings not supported, seems to be an rustpython issue + +# Since = is handled specially, make sure all existing uses of +# it still work. + +self.assertEqual(f'{0==1}', 'False') +self.assertEqual(f'{0!=1}', 'True') +self.assertEqual(f'{0<=1}', 'True') +self.assertEqual(f'{0>=1}', 'False') + +# Make sure leading and following text works. +x = 'foo' +#self.assertEqual(f'X{x=}Y', 'Xx='+repr(x)+'Y') # TODO ' +self.assertEqual(f'X{x=}Y', 'Xx='+x+'Y') # just for the moment + +# Make sure whitespace around the = works. +# self.assertEqual(f'X{x =}Y', 'Xx ='+repr(x)+'Y') # TODO ' +# self.assertEqual(f'X{x= }Y', 'Xx= '+repr(x)+'Y') # TODO ' +# self.assertEqual(f'X{x = }Y', 'Xx = '+repr(x)+'Y') # TODO ' + +# TODO remove trim +self.assertEqual(f'X{x =}Y', 'Xx ='+x+'Y') +self.assertEqual(f'X{x= }Y', 'Xx= '+x+'Y') +self.assertEqual(f'X{x = }Y', 'Xx = '+x+'Y') \ No newline at end of file From c25353c083879906a06e711f0a766959f3448f5b Mon Sep 17 00:00:00 2001 From: TheAnyKey Date: Wed, 29 Apr 2020 15:41:04 -0400 Subject: [PATCH 03/13] minor cleanup --- parser/src/fstring.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index 1bf6a465e5..8ea4985a17 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -83,7 +83,6 @@ impl<'a> FStringParser<'a> { // format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}' '=' if self.chars.peek() != Some(&'=') => { // check for delims empty? pred_expression_text = expression.to_string(); // safe expression before = to print it - // pred_expression_text contains trailing spaces, which are trimmed somewhere later before printing, this needs to be prevented orovercome here } ':' if delims.is_empty() => { From e7ec972ab719c412e6bffc2e06d07d27721521c3 Mon Sep 17 00:00:00 2001 From: TheAnyKey Date: Wed, 29 Apr 2020 16:55:21 -0400 Subject: [PATCH 04/13] now conversion flags can be followed by a format specifier and added tests --- parser/src/fstring.rs | 4 ---- tests/snippets/fstrings.py | 22 ++++++++++++++++------ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index 8ea4985a17..29a588f5e8 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -73,10 +73,6 @@ impl<'a> FStringParser<'a> { return Err(ExpectedRbrace); } }); - - if self.chars.peek() != Some(&'}') { - return Err(ExpectedRbrace); - } } // match a python 3.8 self documenting expression diff --git a/tests/snippets/fstrings.py b/tests/snippets/fstrings.py index 8b1e1d5cd0..76bf91cd1e 100644 --- a/tests/snippets/fstrings.py +++ b/tests/snippets/fstrings.py @@ -71,6 +71,16 @@ def __str__(self): +# Test format specifier after conversion flag +#assert f'{"42"!s:<5}' == '42 ', '#' + f'{"42"!s:5}' +'#' # TODO: default alignment in cpython is left + +assert f'{"42"!s:<5}' == '42 ', '#' + f'{"42"!s:5}' +'#' +assert f'{"42"!s:>5}' == ' 42', '#' + f'{"42"!s:5}' +'#' + +#assert f'{"42"=!s:5}' == '42=42 ', '#'+ f'{"42"!s:5}' +'#' # TODO add ' arround string literal in expression # TODO default alingment in cpython is left +assert f'{"42"=!s:<5}' == '42=42 ', '#'+ f'{"42"!s:5}' +'#' # TODO add ' arround string literal in expression +assert f'{"42"=!s:>5}' == '42= 42', '#'+ f'{"42"!s:5}' +'#' # TODO add ' arround string literal in expression + ### Tests for fstring selfdocumenting form CPython @@ -84,17 +94,17 @@ def assertEqual(self, a,b): x = 'A string' self.assertEqual(f'{10=}', '10=10') self.assertEqual(f'{x=}', 'x=' + x )#repr(x)) # TODO: add ' when printing strings -#self.assertEqual(f'{x =}', 'x =' + x )# + repr(x)) # TODO: remove trim in impl -# self.assertEqual(f'{x=!s}', 'x=' + str(x)) # TODO : implement format specs properly +self.assertEqual(f'{x =}', 'x =' + x )# + repr(x)) # TODO: implement ' handling +self.assertEqual(f'{x=!s}', 'x=' + str(x)) # TODO : implement format specs properly # self.assertEqual(f'{x=!r}', 'x=' + x) #repr(x)) -#self.assertEqual(f'{x=!a}', 'x=' + ascii(x)) +self.assertEqual(f'{x=!a}', 'x=' + ascii(x)) x = 2.71828 self.assertEqual(f'{x=:.2f}', 'x=' + format(x, '.2f')) self.assertEqual(f'{x=:}', 'x=' + format(x, '')) -# self.assertEqual(f'{x=!r:^20}', 'x=' + format(repr(x), '^20')) -# self.assertEqual(f'{x=!s:^20}', 'x=' + format(str(x), '^20')) -# self.assertEqual(f'{x=!a:^20}', 'x=' + format(ascii(x), '^20')) +#self.assertEqual(f'{x=!r:^20}', 'x=' + format(repr(x), '^20')) #TODO formatspecifier after conversion flsg is currently not supported (also for classical fstrings) +#self.assertEqual(f'{x=!s:^20}', 'x=' + format(str(x), '^20')) +#self.assertEqual(f'{x=!a:^20}', 'x=' + format(ascii(x), '^20')) x = 9 self.assertEqual(f'{3*x+15=}', '3*x+15=42') From 972503c3abf13f907bda18e6efd6c39abff6f5fc Mon Sep 17 00:00:00 2001 From: TheAnyKey Date: Wed, 29 Apr 2020 17:13:07 -0400 Subject: [PATCH 05/13] fixed qute handling in tests --- tests/snippets/fstrings.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/snippets/fstrings.py b/tests/snippets/fstrings.py index 76bf91cd1e..6238609c33 100644 --- a/tests/snippets/fstrings.py +++ b/tests/snippets/fstrings.py @@ -74,12 +74,12 @@ def __str__(self): # Test format specifier after conversion flag #assert f'{"42"!s:<5}' == '42 ', '#' + f'{"42"!s:5}' +'#' # TODO: default alignment in cpython is left -assert f'{"42"!s:<5}' == '42 ', '#' + f'{"42"!s:5}' +'#' -assert f'{"42"!s:>5}' == ' 42', '#' + f'{"42"!s:5}' +'#' +assert f'{"42"!s:<5}' == '42 ', '#' + f'{"42"!s:<5}' +'#' +assert f'{"42"!s:>5}' == ' 42', '#' + f'{"42"!s:>5}' +'#' -#assert f'{"42"=!s:5}' == '42=42 ', '#'+ f'{"42"!s:5}' +'#' # TODO add ' arround string literal in expression # TODO default alingment in cpython is left -assert f'{"42"=!s:<5}' == '42=42 ', '#'+ f'{"42"!s:5}' +'#' # TODO add ' arround string literal in expression -assert f'{"42"=!s:>5}' == '42= 42', '#'+ f'{"42"!s:5}' +'#' # TODO add ' arround string literal in expression +#assert f'{"42"=!s:5}' == '42=42 ', '#'+ f'{"42"=!s:5}' +'#' # TODO add ' arround string literal in expression # TODO default alingment in cpython is left +assert f'{"42"=!s:<5}' == '"42"=42 ', '#'+ f'{"42"=!s:<5}' +'#' # TODO add ' arround string literal in expression +assert f'{"42"=!s:>5}' == '"42"= 42', '#'+ f'{"42"=!s:>5}' +'#' # TODO add ' arround string literal in expression From 8e8e839b946eca8b068873d3549050f4f00ae9d4 Mon Sep 17 00:00:00 2001 From: TheAnyKey Date: Wed, 29 Apr 2020 17:16:23 -0400 Subject: [PATCH 06/13] activated further test, removed todos --- tests/snippets/fstrings.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/snippets/fstrings.py b/tests/snippets/fstrings.py index 6238609c33..3e8b48d9b9 100644 --- a/tests/snippets/fstrings.py +++ b/tests/snippets/fstrings.py @@ -77,9 +77,9 @@ def __str__(self): assert f'{"42"!s:<5}' == '42 ', '#' + f'{"42"!s:<5}' +'#' assert f'{"42"!s:>5}' == ' 42', '#' + f'{"42"!s:>5}' +'#' -#assert f'{"42"=!s:5}' == '42=42 ', '#'+ f'{"42"=!s:5}' +'#' # TODO add ' arround string literal in expression # TODO default alingment in cpython is left -assert f'{"42"=!s:<5}' == '"42"=42 ', '#'+ f'{"42"=!s:<5}' +'#' # TODO add ' arround string literal in expression -assert f'{"42"=!s:>5}' == '"42"= 42', '#'+ f'{"42"=!s:>5}' +'#' # TODO add ' arround string literal in expression +#assert f'{"42"=!s:5}' == '"42"=42 ', '#'+ f'{"42"=!s:5}' +'#' # TODO default alingment in cpython is left +assert f'{"42"=!s:<5}' == '"42"=42 ', '#'+ f'{"42"=!s:<5}' +'#' +assert f'{"42"=!s:>5}' == '"42"= 42', '#'+ f'{"42"=!s:>5}' +'#' @@ -95,16 +95,16 @@ def assertEqual(self, a,b): self.assertEqual(f'{10=}', '10=10') self.assertEqual(f'{x=}', 'x=' + x )#repr(x)) # TODO: add ' when printing strings self.assertEqual(f'{x =}', 'x =' + x )# + repr(x)) # TODO: implement ' handling -self.assertEqual(f'{x=!s}', 'x=' + str(x)) # TODO : implement format specs properly +self.assertEqual(f'{x=!s}', 'x=' + str(x)) # self.assertEqual(f'{x=!r}', 'x=' + x) #repr(x)) self.assertEqual(f'{x=!a}', 'x=' + ascii(x)) x = 2.71828 self.assertEqual(f'{x=:.2f}', 'x=' + format(x, '.2f')) self.assertEqual(f'{x=:}', 'x=' + format(x, '')) -#self.assertEqual(f'{x=!r:^20}', 'x=' + format(repr(x), '^20')) #TODO formatspecifier after conversion flsg is currently not supported (also for classical fstrings) -#self.assertEqual(f'{x=!s:^20}', 'x=' + format(str(x), '^20')) -#self.assertEqual(f'{x=!a:^20}', 'x=' + format(ascii(x), '^20')) +self.assertEqual(f'{x=!r:^20}', 'x=' + format(repr(x), '^20')) #TODO formatspecifier after conversion flsg is currently not supported (also for classical fstrings) +self.assertEqual(f'{x=!s:^20}', 'x=' + format(str(x), '^20')) +self.assertEqual(f'{x=!a:^20}', 'x=' + format(ascii(x), '^20')) x = 9 self.assertEqual(f'{3*x+15=}', '3*x+15=42') @@ -146,7 +146,7 @@ def assertEqual(self, a,b): # self.assertEqual(f'X{x= }Y', 'Xx= '+repr(x)+'Y') # TODO ' # self.assertEqual(f'X{x = }Y', 'Xx = '+repr(x)+'Y') # TODO ' -# TODO remove trim + self.assertEqual(f'X{x =}Y', 'Xx ='+x+'Y') self.assertEqual(f'X{x= }Y', 'Xx= '+x+'Y') self.assertEqual(f'X{x = }Y', 'Xx = '+x+'Y') \ No newline at end of file From 3ccb117b9a873ff15295ef734ccc015ef18d0413 Mon Sep 17 00:00:00 2001 From: TheAnyKey Date: Wed, 29 Apr 2020 17:52:47 -0400 Subject: [PATCH 07/13] fixed fmt --- parser/src/fstring.rs | 48 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index 29a588f5e8..093e65a22b 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -27,7 +27,7 @@ impl<'a> FStringParser<'a> { let mut delims = Vec::new(); let mut conversion = None; let mut pred_expression_text = String::new(); - let mut trailing_seq=String::new(); + let mut trailing_seq = String::new(); while let Some(ch) = self.chars.next() { match ch { @@ -77,7 +77,8 @@ impl<'a> FStringParser<'a> { // match a python 3.8 self documenting expression // format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}' - '=' if self.chars.peek() != Some(&'=') => { // check for delims empty? + '=' if self.chars.peek() != Some(&'=') => { + // check for delims empty? pred_expression_text = expression.to_string(); // safe expression before = to print it } @@ -161,28 +162,25 @@ impl<'a> FStringParser<'a> { conversion, spec, }); - } - else { - return Ok(Joined{ - values:vec![ - Constant{ - value:pred_expression_text.to_owned()+"=" - }, - - Constant { - value:trailing_seq.to_owned() - }, - - FormattedValue { + } else { + return Ok(Joined { + values: vec![ + Constant { + value: pred_expression_text.to_owned() + "=", + }, + Constant { + value: trailing_seq.to_owned(), + }, + FormattedValue { value: Box::new( parse_expression(expression.trim()) .map_err(|e| InvalidExpression(Box::new(e.error)))?, ), conversion, - spec,}, - ] - } - ); + spec, + }, + ], + }); } } '"' | '\'' => { @@ -360,24 +358,24 @@ mod tests { #[test] fn test_fstring_parse_selfdocumenting_base() { - let src=String::from("{user=}"); - let parse_ast=parse_fstring(&src); + let src = String::from("{user=}"); + let parse_ast = parse_fstring(&src); assert!(parse_ast.is_ok()); } #[test] fn test_fstring_parse_selfdocumenting_base_more() { - let src=String::from("mix {user=} with text and {second=}"); - let parse_ast=parse_fstring(&src); + let src = String::from("mix {user=} with text and {second=}"); + let parse_ast = parse_fstring(&src); assert!(parse_ast.is_ok()); } #[test] fn test_fstring_parse_selfdocumenting_format() { - let src=String::from("{user=:>10}"); - let parse_ast=parse_fstring(&src); + let src = String::from("{user=:>10}"); + let parse_ast = parse_fstring(&src); assert!(parse_ast.is_ok()); } From 3e8cde7b9c05189d0b595f084d5555442580346f Mon Sep 17 00:00:00 2001 From: TheAnyKey Date: Thu, 30 Apr 2020 12:48:41 -0400 Subject: [PATCH 08/13] Removed incompatibilty with CPython, fixed parser error handling --- parser/src/fstring.rs | 6 ++++++ tests/snippets/fstrings.py | 9 +++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index 093e65a22b..8b226e8e18 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -73,6 +73,11 @@ impl<'a> FStringParser<'a> { return Err(ExpectedRbrace); } }); + + let peek = self.chars.peek(); + if peek != Some(&'}') && peek != Some(&':') { + return Err(ExpectedRbrace); + } } // match a python 3.8 self documenting expression @@ -383,6 +388,7 @@ mod tests { #[test] fn test_parse_invalid_fstring() { assert_eq!(parse_fstring("{5!a"), Err(ExpectedRbrace)); + assert_eq!(parse_fstring("{5!a1}"), Err(ExpectedRbrace)); assert_eq!(parse_fstring("{5!"), Err(ExpectedRbrace)); diff --git a/tests/snippets/fstrings.py b/tests/snippets/fstrings.py index 3e8b48d9b9..bda7496e84 100644 --- a/tests/snippets/fstrings.py +++ b/tests/snippets/fstrings.py @@ -17,10 +17,15 @@ assert f'{16:0>+#10x}' == '00000+0x10' assert f"{{{(lambda x: f'hello, {x}')('world}')}" == '{hello, world}' -assert f'{foo=}' == 'foo=bar' + +# base test of self documenting strings +#assert f'{foo=}' == 'foo=bar' # TODO ' missing num=42 -assert f'{num=}' == 'num=42' + +f'{num=}' # keep this line as it will fail when using a python 3.7 interpreter + +assert f'{num=}' == 'num=42', assert f'{num=:>10}' == 'num= 42' From 80a9710aa8157ccb0b06a5f16156432d4cb0d2d0 Mon Sep 17 00:00:00 2001 From: TheAnyKey Date: Thu, 30 Apr 2020 14:25:49 -0400 Subject: [PATCH 09/13] fixed clippy; tests fail explicitly for python 3.7 and below --- parser/src/fstring.rs | 4 ++-- tests/snippets/fstrings.py | 20 ++++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index 8b226e8e18..ed597b92e9 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -171,10 +171,10 @@ impl<'a> FStringParser<'a> { return Ok(Joined { values: vec![ Constant { - value: pred_expression_text.to_owned() + "=", + value: pred_expression_text + "=", }, Constant { - value: trailing_seq.to_owned(), + value: trailing_seq, }, FormattedValue { value: Box::new( diff --git a/tests/snippets/fstrings.py b/tests/snippets/fstrings.py index bda7496e84..48e82b74d2 100644 --- a/tests/snippets/fstrings.py +++ b/tests/snippets/fstrings.py @@ -1,4 +1,20 @@ -#from testutils import assert_raises +from testutils import assert_raises + +#test only makes sense with python 3.8 or higher (or RustPython) +import sys +import platform +if platform.python_implementation() == 'CPython': + assert sys.version_info.major == 3, 'Incompatible Python Version, expected CPython 3.8 or later' + assert sys.version_info.minor == 8, 'Incompatible Python Version, expected CPython 3.8 or later' +elif platform.python_implementation == 'RustPython': + # ok + pass +else: + # other implementation - lets give it a try + pass + + +# lets start tersing foo = 'bar' assert f"{''}" == '' @@ -25,7 +41,7 @@ f'{num=}' # keep this line as it will fail when using a python 3.7 interpreter -assert f'{num=}' == 'num=42', +assert f'{num=}' == 'num=42' assert f'{num=:>10}' == 'num= 42' From f823af19759c94af37dbd4e6e769e8a7874f246d Mon Sep 17 00:00:00 2001 From: TheAnyKey Date: Thu, 30 Apr 2020 14:51:46 -0400 Subject: [PATCH 10/13] skipping inkompatible test for the moment --- tests/snippets/fstrings.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/snippets/fstrings.py b/tests/snippets/fstrings.py index 48e82b74d2..249fcaeadd 100644 --- a/tests/snippets/fstrings.py +++ b/tests/snippets/fstrings.py @@ -114,11 +114,11 @@ def assertEqual(self, a,b): x = 'A string' self.assertEqual(f'{10=}', '10=10') -self.assertEqual(f'{x=}', 'x=' + x )#repr(x)) # TODO: add ' when printing strings -self.assertEqual(f'{x =}', 'x =' + x )# + repr(x)) # TODO: implement ' handling +# self.assertEqual(f'{x=}', 'x=' + x )#repr(x)) # TODO: add ' when printing strings +# self.assertEqual(f'{x =}', 'x =' + x )# + repr(x)) # TODO: implement ' handling self.assertEqual(f'{x=!s}', 'x=' + str(x)) -# self.assertEqual(f'{x=!r}', 'x=' + x) #repr(x)) -self.assertEqual(f'{x=!a}', 'x=' + ascii(x)) +# # self.assertEqual(f'{x=!r}', 'x=' + x) #repr(x)) # !r not supported +# self.assertEqual(f'{x=!a}', 'x=' + ascii(x)) x = 2.71828 self.assertEqual(f'{x=:.2f}', 'x=' + format(x, '.2f')) @@ -158,9 +158,9 @@ def assertEqual(self, a,b): self.assertEqual(f'{0>=1}', 'False') # Make sure leading and following text works. -x = 'foo' +# x = 'foo' #self.assertEqual(f'X{x=}Y', 'Xx='+repr(x)+'Y') # TODO ' -self.assertEqual(f'X{x=}Y', 'Xx='+x+'Y') # just for the moment +# self.assertEqual(f'X{x=}Y', 'Xx='+x+'Y') # just for the moment # Make sure whitespace around the = works. # self.assertEqual(f'X{x =}Y', 'Xx ='+repr(x)+'Y') # TODO ' @@ -168,6 +168,6 @@ def assertEqual(self, a,b): # self.assertEqual(f'X{x = }Y', 'Xx = '+repr(x)+'Y') # TODO ' -self.assertEqual(f'X{x =}Y', 'Xx ='+x+'Y') -self.assertEqual(f'X{x= }Y', 'Xx= '+x+'Y') -self.assertEqual(f'X{x = }Y', 'Xx = '+x+'Y') \ No newline at end of file +# self.assertEqual(f'X{x =}Y', 'Xx ='+x+'Y') +# self.assertEqual(f'X{x= }Y', 'Xx= '+x+'Y') +# self.assertEqual(f'X{x = }Y', 'Xx = '+x+'Y') \ No newline at end of file From 620115d69a831d95e744b10231d3520023b4cde7 Mon Sep 17 00:00:00 2001 From: TheAnyKey Date: Fri, 1 May 2020 10:14:02 -0400 Subject: [PATCH 11/13] recommit --- tests/snippets/fstrings.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/snippets/fstrings.py b/tests/snippets/fstrings.py index 249fcaeadd..b72e6b796c 100644 --- a/tests/snippets/fstrings.py +++ b/tests/snippets/fstrings.py @@ -167,7 +167,6 @@ def assertEqual(self, a,b): # self.assertEqual(f'X{x= }Y', 'Xx= '+repr(x)+'Y') # TODO ' # self.assertEqual(f'X{x = }Y', 'Xx = '+repr(x)+'Y') # TODO ' - # self.assertEqual(f'X{x =}Y', 'Xx ='+x+'Y') # self.assertEqual(f'X{x= }Y', 'Xx= '+x+'Y') # self.assertEqual(f'X{x = }Y', 'Xx = '+x+'Y') \ No newline at end of file From cba961ecfae66ba78a390dc2c89e04daeb711b7c Mon Sep 17 00:00:00 2001 From: TheAnyKey <32773684+TheAnyKey@users.noreply.github.com> Date: Mon, 4 May 2020 23:09:31 +0200 Subject: [PATCH 12/13] Apply suggestions from code review Co-authored-by: Jeong YunWon --- parser/src/fstring.rs | 10 +++------- tests/snippets/fstrings.py | 5 ++--- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index ed597b92e9..a3a3c046d8 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -34,8 +34,7 @@ impl<'a> FStringParser<'a> { // can be integrated better with the remainign code, but as a starting point ok // in general I would do here a tokenizing of the fstrings to omit this peeking. '!' if self.chars.peek() == Some(&'=') => { - expression.push('!'); - expression.push('='); + expression.push_str("!="); self.chars.next(); } @@ -74,8 +73,8 @@ impl<'a> FStringParser<'a> { } }); - let peek = self.chars.peek(); - if peek != Some(&'}') && peek != Some(&':') { + if let Some(peek) = self.chars.peek() { + if peek != '}' && peek != ':' { return Err(ExpectedRbrace); } } @@ -197,11 +196,9 @@ impl<'a> FStringParser<'a> { } } } - ' ' if !pred_expression_text.is_empty() => { trailing_seq.push(ch); } - _ => { expression.push(ch); } @@ -388,7 +385,6 @@ mod tests { #[test] fn test_parse_invalid_fstring() { assert_eq!(parse_fstring("{5!a"), Err(ExpectedRbrace)); - assert_eq!(parse_fstring("{5!a1}"), Err(ExpectedRbrace)); assert_eq!(parse_fstring("{5!"), Err(ExpectedRbrace)); diff --git a/tests/snippets/fstrings.py b/tests/snippets/fstrings.py index b72e6b796c..1b7ef9c421 100644 --- a/tests/snippets/fstrings.py +++ b/tests/snippets/fstrings.py @@ -4,8 +4,7 @@ import sys import platform if platform.python_implementation() == 'CPython': - assert sys.version_info.major == 3, 'Incompatible Python Version, expected CPython 3.8 or later' - assert sys.version_info.minor == 8, 'Incompatible Python Version, expected CPython 3.8 or later' + assert sys.version_info >= (3, 8), 'Incompatible Python Version, expected CPython 3.8 or later' elif platform.python_implementation == 'RustPython': # ok pass @@ -169,4 +168,4 @@ def assertEqual(self, a,b): # self.assertEqual(f'X{x =}Y', 'Xx ='+x+'Y') # self.assertEqual(f'X{x= }Y', 'Xx= '+x+'Y') -# self.assertEqual(f'X{x = }Y', 'Xx = '+x+'Y') \ No newline at end of file +# self.assertEqual(f'X{x = }Y', 'Xx = '+x+'Y') From 71ec5c1b86034ef0ce551b539500da796ac1eba0 Mon Sep 17 00:00:00 2001 From: TheAnyKey Date: Tue, 5 May 2020 05:01:16 -0400 Subject: [PATCH 13/13] clean up after review --- parser/src/fstring.rs | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs index ed597b92e9..6f399c681f 100644 --- a/parser/src/fstring.rs +++ b/parser/src/fstring.rs @@ -34,26 +34,22 @@ impl<'a> FStringParser<'a> { // can be integrated better with the remainign code, but as a starting point ok // in general I would do here a tokenizing of the fstrings to omit this peeking. '!' if self.chars.peek() == Some(&'=') => { - expression.push('!'); - expression.push('='); + expression.push_str("!="); self.chars.next(); } '=' if self.chars.peek() == Some(&'=') => { - expression.push('='); - expression.push('='); + expression.push_str("=="); self.chars.next(); } '>' if self.chars.peek() == Some(&'=') => { - expression.push('>'); - expression.push('='); + expression.push_str(">="); self.chars.next(); } '<' if self.chars.peek() == Some(&'=') => { - expression.push('<'); - expression.push('='); + expression.push_str("<="); self.chars.next(); } @@ -74,8 +70,11 @@ impl<'a> FStringParser<'a> { } }); - let peek = self.chars.peek(); - if peek != Some(&'}') && peek != Some(&':') { + if let Some(&peek) = self.chars.peek() { + if peek != '}' && peek != ':' { + return Err(ExpectedRbrace); + } + } else { return Err(ExpectedRbrace); } } @@ -197,17 +196,14 @@ impl<'a> FStringParser<'a> { } } } - ' ' if !pred_expression_text.is_empty() => { trailing_seq.push(ch); } - _ => { expression.push(ch); } } } - Err(UnclosedLbrace) } @@ -391,7 +387,6 @@ mod tests { assert_eq!(parse_fstring("{5!a1}"), Err(ExpectedRbrace)); assert_eq!(parse_fstring("{5!"), Err(ExpectedRbrace)); - assert_eq!(parse_fstring("abc{!a 'cat'}"), Err(EmptyExpression)); assert_eq!(parse_fstring("{!a"), Err(EmptyExpression)); assert_eq!(parse_fstring("{ !a}"), Err(EmptyExpression));