From e354bc8e78f0a3d156bf6e8670f79a9b18bf19bd Mon Sep 17 00:00:00 2001 From: hydrogen602 Date: Sat, 20 Apr 2024 20:38:30 -0500 Subject: [PATCH 1/7] compile accepts memoryview now --- Lib/test/test_builtin.py | 2 -- vm/src/stdlib/builtins.rs | 23 +++++++++++++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 74f61a26e1..a35e80850e 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -327,8 +327,6 @@ def test_chr(self): def test_cmp(self): self.assertTrue(not hasattr(builtins, "cmp")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_compile(self): compile('print(1)\n', '', 'exec') bom = b'\xef\xbb\xbf' diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index fbe1170c03..dfb514d7f4 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -146,16 +146,35 @@ mod builtins { } #[cfg(feature = "rustpython-parser")] { + use crate::builtins::PyMemoryView; + use crate::common::borrow::BorrowedValue; + use crate::protocol::PyBuffer; + use crate::types::AsBuffer; use crate::{builtins::PyBytesRef, convert::ToPyException}; use num_traits::Zero; use rustpython_parser as parser; - let source = Either::::try_from_object(vm, args.source)?; + // + let source = + Either::>>::try_from_object( + vm, + args.source, + )?; + + let memory_view_holder: PyBuffer; + let memory_view_borrow_holder: BorrowedValue<[u8]>; + // TODO: compiler::compile should probably get bytes let source = match &source { Either::A(string) => string.as_str(), - Either::B(bytes) => std::str::from_utf8(bytes) + Either::B(Either::A(bytes)) => std::str::from_utf8(bytes) .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?, + Either::B(Either::B(memory_view)) => { + memory_view_holder = AsBuffer::as_buffer(memory_view, vm)?; + memory_view_borrow_holder = memory_view_holder.obj_bytes(); + std::str::from_utf8(memory_view_borrow_holder.as_ref()) + .map_err(|e| vm.new_unicode_decode_error(e.to_string()))? + } }; let flags = args.flags.map_or(Ok(0), |v| v.try_to_primitive(vm))?; From f5890d1e99db6b759f8b3e0cc4bd198212b487f5 Mon Sep 17 00:00:00 2001 From: hydrogen602 Date: Sun, 21 Apr 2024 01:24:06 -0500 Subject: [PATCH 2/7] Update test_compile to what it is in CPython3.12 --- Lib/test/test_builtin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index a35e80850e..f6ae8ee5ad 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -338,11 +338,10 @@ def test_compile(self): self.assertRaises(TypeError, compile) self.assertRaises(ValueError, compile, 'print(42)\n', '', 'badmode') self.assertRaises(ValueError, compile, 'print(42)\n', '', 'single', 0xff) - self.assertRaises(ValueError, compile, chr(0), 'f', 'exec') self.assertRaises(TypeError, compile, 'pass', '?', 'exec', mode='eval', source='0', filename='tmp') compile('print("\xe5")\n', '', 'exec') - self.assertRaises(ValueError, compile, chr(0), 'f', 'exec') + self.assertRaises(SyntaxError, compile, chr(0), 'f', 'exec') self.assertRaises(ValueError, compile, str('a = 1'), 'f', 'bad') # test the optimize argument From fb8cc97f958cfb301345aadafeded12823d96f7f Mon Sep 17 00:00:00 2001 From: hydrogen602 Date: Sun, 21 Apr 2024 01:25:30 -0500 Subject: [PATCH 3/7] flag check fix --- vm/src/stdlib/ast.rs | 2 ++ vm/src/stdlib/builtins.rs | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index 50b3153d0c..b520e9a5a7 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -353,6 +353,8 @@ pub(crate) fn compile( pub(crate) use _ast::NodeAst; // Used by builtins::compile() pub const PY_COMPILE_FLAG_AST_ONLY: i32 = 0x0400; +// Used by builtins::compile() - the summary of all flags +pub const PY_COMPILE_FLAGS_MASK: i32 = PY_COMPILE_FLAG_AST_ONLY; pub fn make_module(vm: &VirtualMachine) -> PyRef { let module = _ast::make_module(vm); diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index dfb514d7f4..3811823b05 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -179,6 +179,10 @@ mod builtins { let flags = args.flags.map_or(Ok(0), |v| v.try_to_primitive(vm))?; + if !(flags & !ast::PY_COMPILE_FLAGS_MASK).is_zero() { + return Err(vm.new_value_error("compile() unrecognized flags".to_owned())); + } + if (flags & ast::PY_COMPILE_FLAG_AST_ONLY).is_zero() { #[cfg(not(feature = "rustpython-compiler"))] { From f96ecc9a537cf65e61b5eb48d19e0655d8dd3e50 Mon Sep 17 00:00:00 2001 From: hydrogen602 Date: Sun, 21 Apr 2024 02:03:15 -0500 Subject: [PATCH 4/7] compile optimize flag --- Lib/test/test_builtin.py | 2 +- vm/src/stdlib/ast.rs | 7 ++++++- vm/src/stdlib/builtins.rs | 28 ++++++++++++++++++++++++++-- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index f6ae8ee5ad..23590140dd 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -373,7 +373,7 @@ def f(): """doc""" ns = {} exec(code, ns) rv = ns['f']() - self.assertEqual(rv, tuple(expected)) + self.assertEqual(rv, tuple(expected), msg=f'optval={optval}') # TODO: RUSTPYTHON @unittest.expectedFailure diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index b520e9a5a7..acd7865b15 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -341,8 +341,13 @@ pub(crate) fn compile( object: PyObjectRef, filename: &str, mode: crate::compiler::Mode, + optimize: Option, ) -> PyResult { - let opts = vm.compile_opts(); + let mut opts = vm.compile_opts(); + if let Some(optimize) = optimize { + opts.optimize = optimize; + } + let ast = Node::ast_from_object(vm, object)?; let code = codegen::compile::compile_top(&ast, filename.to_owned(), mode, opts) .map_err(|err| (CompileError::from(err), None).to_pyexception(vm))?; // FIXME source diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index 3811823b05..108270490d 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -121,6 +121,15 @@ mod builtins { let mode_str = args.mode.as_str(); + let optimize: i32 = args.optimize.map_or(Ok(-1), |v| v.try_to_primitive(vm))?; + let optimize: u8 = if optimize == -1 { + vm.state.settings.optimize + } else { + optimize.try_into().map_err(|_| { + vm.new_value_error("compile() optimize value invalid".to_owned()) + })? + }; + if args .source .fast_isinstance(&ast::NodeAst::make_class(&vm.ctx)) @@ -134,7 +143,13 @@ mod builtins { let mode = mode_str .parse::() .map_err(|err| vm.new_value_error(err.to_string()))?; - return ast::compile(vm, args.source, args.filename.as_str(), mode); + return ast::compile( + vm, + args.source, + args.filename.as_str(), + mode, + Some(optimize), + ); } } @@ -193,8 +208,17 @@ mod builtins { let mode = mode_str .parse::() .map_err(|err| vm.new_value_error(err.to_string()))?; + + let mut opts = vm.compile_opts(); + opts.optimize = optimize; + let code = vm - .compile(source, mode, args.filename.as_str().to_owned()) + .compile_with_opts( + source, + mode, + args.filename.as_str().to_owned(), + opts, + ) .map_err(|err| (err, Some(source)).to_pyexception(vm))?; Ok(code.into()) } From 3a5150a1047fd860f66ed2a9f0c37226f203a559 Mon Sep 17 00:00:00 2001 From: hydrogen602 Date: Mon, 22 Apr 2024 20:04:53 -0500 Subject: [PATCH 5/7] address PR comments --- Lib/test/test_builtin.py | 4 +++- vm/src/stdlib/builtins.rs | 30 +++++------------------------- 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 23590140dd..fe64006613 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -327,6 +327,8 @@ def test_chr(self): def test_cmp(self): self.assertTrue(not hasattr(builtins, "cmp")) + # TODO: RUSTPYTHON optval=2 does not remove docstrings + @unittest.expectedFailure def test_compile(self): compile('print(1)\n', '', 'exec') bom = b'\xef\xbb\xbf' @@ -373,7 +375,7 @@ def f(): """doc""" ns = {} exec(code, ns) rv = ns['f']() - self.assertEqual(rv, tuple(expected), msg=f'optval={optval}') + self.assertEqual(rv, tuple(expected)) # TODO: RUSTPYTHON @unittest.expectedFailure diff --git a/vm/src/stdlib/builtins.rs b/vm/src/stdlib/builtins.rs index 108270490d..bdbdc8b5ed 100644 --- a/vm/src/stdlib/builtins.rs +++ b/vm/src/stdlib/builtins.rs @@ -161,36 +161,16 @@ mod builtins { } #[cfg(feature = "rustpython-parser")] { - use crate::builtins::PyMemoryView; - use crate::common::borrow::BorrowedValue; - use crate::protocol::PyBuffer; - use crate::types::AsBuffer; - use crate::{builtins::PyBytesRef, convert::ToPyException}; + use crate::convert::ToPyException; use num_traits::Zero; use rustpython_parser as parser; - // - let source = - Either::>>::try_from_object( - vm, - args.source, - )?; - - let memory_view_holder: PyBuffer; - let memory_view_borrow_holder: BorrowedValue<[u8]>; + let source = ArgStrOrBytesLike::try_from_object(vm, args.source)?; + let source = source.borrow_bytes(); // TODO: compiler::compile should probably get bytes - let source = match &source { - Either::A(string) => string.as_str(), - Either::B(Either::A(bytes)) => std::str::from_utf8(bytes) - .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?, - Either::B(Either::B(memory_view)) => { - memory_view_holder = AsBuffer::as_buffer(memory_view, vm)?; - memory_view_borrow_holder = memory_view_holder.obj_bytes(); - std::str::from_utf8(memory_view_borrow_holder.as_ref()) - .map_err(|e| vm.new_unicode_decode_error(e.to_string()))? - } - }; + let source = std::str::from_utf8(&source) + .map_err(|e| vm.new_unicode_decode_error(e.to_string()))?; let flags = args.flags.map_or(Ok(0), |v| v.try_to_primitive(vm))?; From b634a7b8eadfd9969f265b29bccf7aca3531a1bd Mon Sep 17 00:00:00 2001 From: hydrogen602 Date: Mon, 22 Apr 2024 21:46:05 -0500 Subject: [PATCH 6/7] added undocumented flags to flag validator --- vm/src/stdlib/ast.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index acd7865b15..b86a6798cd 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -358,8 +358,16 @@ pub(crate) fn compile( pub(crate) use _ast::NodeAst; // Used by builtins::compile() pub const PY_COMPILE_FLAG_AST_ONLY: i32 = 0x0400; + +// The following flags match the values from Include/cpython/compile.h +// Caveat emptor: These flags are undocumented on purpose and depending +// on their effect outside the standard library is **unsupported**. +const PY_CF_DONT_IMPLY_DEDENT: i32 = 0x200; +const PY_CF_ALLOW_INCOMPLETE_INPUT: i32 = 0x4000; + // Used by builtins::compile() - the summary of all flags -pub const PY_COMPILE_FLAGS_MASK: i32 = PY_COMPILE_FLAG_AST_ONLY; +pub const PY_COMPILE_FLAGS_MASK: i32 = + PY_COMPILE_FLAG_AST_ONLY | PY_CF_DONT_IMPLY_DEDENT | PY_CF_ALLOW_INCOMPLETE_INPUT; pub fn make_module(vm: &VirtualMachine) -> PyRef { let module = _ast::make_module(vm); From 56cdf69b02b051f5c8d6ec9566d044b737553239 Mon Sep 17 00:00:00 2001 From: hydrogen602 Date: Mon, 22 Apr 2024 22:49:33 -0500 Subject: [PATCH 7/7] flags added to filter --- vm/src/stdlib/ast.rs | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/ast.rs b/vm/src/stdlib/ast.rs index b86a6798cd..8b081294b9 100644 --- a/vm/src/stdlib/ast.rs +++ b/vm/src/stdlib/ast.rs @@ -365,9 +365,36 @@ pub const PY_COMPILE_FLAG_AST_ONLY: i32 = 0x0400; const PY_CF_DONT_IMPLY_DEDENT: i32 = 0x200; const PY_CF_ALLOW_INCOMPLETE_INPUT: i32 = 0x4000; +// __future__ flags - sync with Lib/__future__.py +// TODO: These flags aren't being used in rust code +// CO_FUTURE_ANNOTATIONS does make a difference in the codegen, +// so it should be used in compile(). +// see compiler/codegen/src/compile.rs +const CO_NESTED: i32 = 0x0010; +const CO_GENERATOR_ALLOWED: i32 = 0; +const CO_FUTURE_DIVISION: i32 = 0x20000; +const CO_FUTURE_ABSOLUTE_IMPORT: i32 = 0x40000; +const CO_FUTURE_WITH_STATEMENT: i32 = 0x80000; +const CO_FUTURE_PRINT_FUNCTION: i32 = 0x100000; +const CO_FUTURE_UNICODE_LITERALS: i32 = 0x200000; +const CO_FUTURE_BARRY_AS_BDFL: i32 = 0x400000; +const CO_FUTURE_GENERATOR_STOP: i32 = 0x800000; +const CO_FUTURE_ANNOTATIONS: i32 = 0x1000000; + // Used by builtins::compile() - the summary of all flags -pub const PY_COMPILE_FLAGS_MASK: i32 = - PY_COMPILE_FLAG_AST_ONLY | PY_CF_DONT_IMPLY_DEDENT | PY_CF_ALLOW_INCOMPLETE_INPUT; +pub const PY_COMPILE_FLAGS_MASK: i32 = PY_COMPILE_FLAG_AST_ONLY + | PY_CF_DONT_IMPLY_DEDENT + | PY_CF_ALLOW_INCOMPLETE_INPUT + | CO_NESTED + | CO_GENERATOR_ALLOWED + | CO_FUTURE_DIVISION + | CO_FUTURE_ABSOLUTE_IMPORT + | CO_FUTURE_WITH_STATEMENT + | CO_FUTURE_PRINT_FUNCTION + | CO_FUTURE_UNICODE_LITERALS + | CO_FUTURE_BARRY_AS_BDFL + | CO_FUTURE_GENERATOR_STOP + | CO_FUTURE_ANNOTATIONS; pub fn make_module(vm: &VirtualMachine) -> PyRef { let module = _ast::make_module(vm);