From 3954dbe716bbfa4d1802aabd9cd9e8517d3fb552 Mon Sep 17 00:00:00 2001 From: Windel Bouwman Date: Sun, 30 Jun 2019 20:11:40 +0200 Subject: [PATCH] Make parser and compiler optional features for vm crate. --- Cargo.lock | 2 ++ Cargo.toml | 1 + src/main.rs | 5 ++--- vm/Cargo.toml | 7 +++++-- vm/src/builtins.rs | 18 ++++++++++++++---- vm/src/eval.rs | 2 +- vm/src/import.rs | 9 +++++++-- vm/src/lib.rs | 2 +- vm/src/stdlib/mod.rs | 21 ++++++++++++++++----- vm/src/vm.rs | 8 ++++++-- wasm/lib/Cargo.toml | 1 + wasm/lib/src/vm_class.rs | 6 ++---- 12 files changed, 58 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2d24a53b9c..af153b7c36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -952,6 +952,7 @@ dependencies = [ "cpython 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "rustpython_compiler 0.1.0", "rustpython_parser 0.0.1", "rustpython_vm 0.1.0", "rustyline 4.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1061,6 +1062,7 @@ dependencies = [ "futures 0.1.27 (registry+https://github.com/rust-lang/crates.io-index)", "js-sys 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)", "num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rustpython_compiler 0.1.0", "rustpython_parser 0.0.1", "rustpython_vm 0.1.0", "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Cargo.toml b/Cargo.toml index 315b5114c0..45c5c940d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ path = "./benchmarks/bench.rs" log="0.4.1" env_logger="0.5.10" clap = "2.31.2" +rustpython_compiler = {path = "compiler"} rustpython_parser = {path = "parser"} rustpython_vm = {path = "vm"} rustyline = "4.1.0" diff --git a/src/main.rs b/src/main.rs index 5d4d2b9e72..bcd8c9ccd3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,11 +8,9 @@ extern crate rustpython_vm; extern crate rustyline; use clap::{App, Arg}; +use rustpython_compiler::{compile, error::CompileError, error::CompileErrorType}; use rustpython_parser::error::ParseError; use rustpython_vm::{ - compile, - error::CompileError, - error::CompileErrorType, frame::Scope, import, obj::objstr, @@ -20,6 +18,7 @@ use rustpython_vm::{ pyobject::{ItemProtocol, PyResult}, util, VirtualMachine, }; + use rustyline::{error::ReadlineError, Editor}; use std::path::PathBuf; diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 6e1e800346..021e7f5b13 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -4,6 +4,9 @@ version = "0.1.0" authors = ["Shing Lyu "] edition = "2018" +[features] +default = ["rustpython_parser", "rustpython_compiler"] + [dependencies] # Crypto: digest = "0.8.1" @@ -21,8 +24,8 @@ num-rational = "0.2.1" rand = "0.5" log = "0.3" rustpython_derive = {path = "../derive"} -rustpython_parser = {path = "../parser"} -rustpython_compiler = {path = "../compiler"} +rustpython_parser = {path = "../parser", optional = true} +rustpython_compiler = {path = "../compiler", optional = true} rustpython_bytecode = { path = "../bytecode" } serde = { version = "1.0.66", features = ["derive"] } serde_json = "1.0.26" diff --git a/vm/src/builtins.rs b/vm/src/builtins.rs index 9734e40218..8c76790cee 100644 --- a/vm/src/builtins.rs +++ b/vm/src/builtins.rs @@ -10,7 +10,6 @@ use std::str; use num_bigint::Sign; use num_traits::{Signed, ToPrimitive, Zero}; -use crate::compile; use crate::obj::objbool; use crate::obj::objbytes::PyBytesRef; use crate::obj::objcode::PyCodeRef; @@ -19,6 +18,8 @@ use crate::obj::objint::{self, PyIntRef}; use crate::obj::objiter; use crate::obj::objstr::{self, PyString, PyStringRef}; use crate::obj::objtype::{self, PyClassRef}; +#[cfg(feature = "rustpython_compiler")] +use rustpython_compiler::compile; use crate::frame::Scope; use crate::function::{single_or_tuple_any, Args, KwArgs, OptionalArg, PyFuncArgs}; @@ -98,6 +99,7 @@ struct CompileArgs { optimize: OptionalArg, } +#[cfg(feature = "rustpython_compiler")] fn builtin_compile(args: CompileArgs, vm: &VirtualMachine) -> PyResult { // TODO: compile::compile should probably get bytes let source = match args.source { @@ -153,6 +155,7 @@ fn builtin_divmod(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { /// Implements `eval`. /// See also: https://docs.python.org/3/library/functions.html#eval +#[cfg(feature = "rustpython_compiler")] fn builtin_eval(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { // TODO: support any mapping for `locals` arg_check!( @@ -184,6 +187,7 @@ fn builtin_eval(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { /// Implements `exec` /// https://docs.python.org/3/library/functions.html#exec +#[cfg(feature = "rustpython_compiler")] fn builtin_exec(vm: &VirtualMachine, args: PyFuncArgs) -> PyResult { arg_check!( vm, @@ -785,6 +789,15 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { #[cfg(not(target_arch = "wasm32"))] let open = vm.ctx.new_rustfunc(io_open); + #[cfg(feature = "rustpython_compiler")] + { + extend_module!(vm, module, { + "compile" => ctx.new_rustfunc(builtin_compile), + "eval" => ctx.new_rustfunc(builtin_eval), + "exec" => ctx.new_rustfunc(builtin_exec), + }); + } + extend_module!(vm, module, { //set __name__ fixes: https://github.com/RustPython/RustPython/issues/146 "__name__" => ctx.new_str(String::from("__main__")), @@ -799,15 +812,12 @@ pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { "callable" => ctx.new_rustfunc(builtin_callable), "chr" => ctx.new_rustfunc(builtin_chr), "classmethod" => ctx.classmethod_type(), - "compile" => ctx.new_rustfunc(builtin_compile), "complex" => ctx.complex_type(), "delattr" => ctx.new_rustfunc(builtin_delattr), "dict" => ctx.dict_type(), "divmod" => ctx.new_rustfunc(builtin_divmod), "dir" => ctx.new_rustfunc(builtin_dir), "enumerate" => ctx.enumerate_type(), - "eval" => ctx.new_rustfunc(builtin_eval), - "exec" => ctx.new_rustfunc(builtin_exec), "float" => ctx.float_type(), "frozenset" => ctx.frozenset_type(), "filter" => ctx.filter_type(), diff --git a/vm/src/eval.rs b/vm/src/eval.rs index eec190d2cc..f8ca7bb7b3 100644 --- a/vm/src/eval.rs +++ b/vm/src/eval.rs @@ -1,7 +1,7 @@ -use crate::compile; use crate::frame::Scope; use crate::pyobject::PyResult; use crate::vm::VirtualMachine; +use rustpython_compiler::compile; pub fn eval(vm: &VirtualMachine, source: &str, scope: Scope, source_path: &str) -> PyResult { match vm.compile(source, &compile::Mode::Eval, source_path.to_string()) { diff --git a/vm/src/import.rs b/vm/src/import.rs index 8b56b23b0e..d6c5e55769 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -5,12 +5,13 @@ use std::path::PathBuf; use crate::bytecode::CodeObject; -use crate::compile; use crate::frame::Scope; use crate::obj::{objcode, objsequence, objstr}; use crate::pyobject::{ItemProtocol, PyResult, PyValue}; use crate::util; use crate::vm::VirtualMachine; +#[cfg(feature = "rustpython_compiler")] +use rustpython_compiler::compile; pub fn init_importlib(vm: &VirtualMachine) -> PyResult { let importlib = import_frozen(vm, "_frozen_importlib")?; @@ -56,7 +57,7 @@ pub fn import_module(vm: &VirtualMachine, current_path: PathBuf, module_name: &s import_frozen(vm, module_name) } else if vm.stdlib_inits.borrow().contains_key(module_name) { import_builtin(vm, module_name) - } else { + } else if cfg!(feature = "rustpython_compiler") { let notfound_error = &vm.ctx.exceptions.module_not_found_error; let import_error = &vm.ctx.exceptions.import_error; @@ -72,9 +73,13 @@ pub fn import_module(vm: &VirtualMachine, current_path: PathBuf, module_name: &s file_path.to_str().unwrap().to_string(), source, ) + } else { + let notfound_error = &vm.ctx.exceptions.module_not_found_error; + Err(vm.new_exception(notfound_error.clone(), module_name.to_string())) } } +#[cfg(feature = "rustpython_compiler")] pub fn import_file( vm: &VirtualMachine, module_name: &str, diff --git a/vm/src/lib.rs b/vm/src/lib.rs index bfd9a66bc1..b68276b904 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -44,6 +44,7 @@ pub mod macros; mod builtins; pub mod cformat; mod dictdatatype; +#[cfg(feature = "rustpython_compiler")] pub mod eval; mod exceptions; pub mod format; @@ -65,7 +66,6 @@ mod vm; pub use self::exceptions::print_exception; pub use self::vm::VirtualMachine; pub use rustpython_bytecode::*; -pub use rustpython_compiler::*; #[doc(hidden)] pub mod __exports { diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index f62d073d42..f853598a67 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -1,3 +1,4 @@ +#[cfg(feature = "rustpython_parser")] mod ast; mod binascii; mod dis; @@ -5,6 +6,7 @@ mod hashlib; mod imp; mod itertools; mod json; +#[cfg(feature = "rustpython_parser")] mod keyword; mod marshal; mod math; @@ -16,6 +18,7 @@ pub mod socket; mod string; mod thread; mod time_module; +#[cfg(feature = "rustpython_parser")] mod tokenize; mod warnings; mod weakref; @@ -37,13 +40,11 @@ pub type StdlibInitFunc = Box PyObjectRef>; pub fn get_module_inits() -> HashMap { #[allow(unused_mut)] let mut modules = hashmap! { - "ast".to_string() => Box::new(ast::make_module) as StdlibInitFunc, - "binascii".to_string() => Box::new(binascii::make_module), - "dis".to_string() => Box::new(dis::make_module), + "binascii".to_string() => Box::new(binascii::make_module) as StdlibInitFunc, + "dis".to_string() => Box::new(dis::make_module) as StdlibInitFunc, "hashlib".to_string() => Box::new(hashlib::make_module), "itertools".to_string() => Box::new(itertools::make_module), "json".to_string() => Box::new(json::make_module), - "keyword".to_string() => Box::new(keyword::make_module), "marshal".to_string() => Box::new(marshal::make_module), "math".to_string() => Box::new(math::make_module), "platform".to_string() => Box::new(platform::make_module), @@ -53,12 +54,22 @@ pub fn get_module_inits() -> HashMap { "struct".to_string() => Box::new(pystruct::make_module), "_thread".to_string() => Box::new(thread::make_module), "time".to_string() => Box::new(time_module::make_module), - "tokenize".to_string() => Box::new(tokenize::make_module), "_weakref".to_string() => Box::new(weakref::make_module), "_imp".to_string() => Box::new(imp::make_module), "_warnings".to_string() => Box::new(warnings::make_module), }; + // Insert parser related modules: + #[cfg(feature = "rustpython_parser")] + { + modules.insert( + "ast".to_string(), + Box::new(ast::make_module) as StdlibInitFunc, + ); + modules.insert("keyword".to_string(), Box::new(keyword::make_module)); + modules.insert("tokenize".to_string(), Box::new(tokenize::make_module)); + } + // disable some modules on WASM #[cfg(not(target_arch = "wasm32"))] { diff --git a/vm/src/vm.rs b/vm/src/vm.rs index c8a1315523..632443c8f0 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -12,8 +12,6 @@ use std::sync::{Mutex, MutexGuard}; use crate::builtins; use crate::bytecode; -use crate::compile; -use crate::error::CompileError; use crate::frame::{ExecutionResult, Frame, FrameRef, Scope}; use crate::frozen; use crate::function::PyFuncArgs; @@ -38,6 +36,10 @@ use crate::pyobject::{ use crate::stdlib; use crate::sysmodule; use num_bigint::BigInt; +#[cfg(feature = "rustpython_compiler")] +use rustpython_compiler::compile; +#[cfg(feature = "rustpython_compiler")] +use rustpython_compiler::error::CompileError; // use objects::objects; @@ -249,6 +251,7 @@ impl VirtualMachine { self.new_exception(overflow_error, msg) } + #[cfg(feature = "rustpython_compiler")] pub fn new_syntax_error(&self, error: &CompileError) -> PyObjectRef { let syntax_error_type = self.ctx.exceptions.syntax_error.clone(); let syntax_error = self.new_exception(syntax_error_type, error.to_string()); @@ -727,6 +730,7 @@ impl VirtualMachine { ) } + #[cfg(feature = "rustpython_compiler")] pub fn compile( &self, source: &str, diff --git a/wasm/lib/Cargo.toml b/wasm/lib/Cargo.toml index 3e045ae876..35f02ba8d0 100644 --- a/wasm/lib/Cargo.toml +++ b/wasm/lib/Cargo.toml @@ -11,6 +11,7 @@ edition = "2018" crate-type = ["cdylib", "rlib"] [dependencies] +rustpython_compiler = { path = "../../compiler" } rustpython_parser = { path = "../../parser" } rustpython_vm = { path = "../../vm" } cfg-if = "0.1.2" diff --git a/wasm/lib/src/vm_class.rs b/wasm/lib/src/vm_class.rs index 886a7013e2..1500304bc2 100644 --- a/wasm/lib/src/vm_class.rs +++ b/wasm/lib/src/vm_class.rs @@ -5,7 +5,7 @@ use std::rc::{Rc, Weak}; use js_sys::{Object, Reflect, SyntaxError, TypeError}; use wasm_bindgen::prelude::*; -use rustpython_vm::compile; +use rustpython_compiler::{compile, error::CompileErrorType}; use rustpython_vm::frame::{NameProtocol, Scope}; use rustpython_vm::function::PyFuncArgs; use rustpython_vm::pyobject::{PyObject, PyObjectPayload, PyObjectRef, PyResult, PyValue}; @@ -280,9 +280,7 @@ impl WASMVirtualMachine { let code = vm.compile(&source, &mode, "".to_string()); let code = code.map_err(|err| { let js_err = SyntaxError::new(&format!("Error parsing Python code: {}", err)); - if let rustpython_vm::error::CompileErrorType::Parse(ref parse_error) = - err.error - { + if let CompileErrorType::Parse(ref parse_error) = err.error { use rustpython_parser::error::ParseError; if let ParseError::EOF(Some(ref loc)) | ParseError::ExtraToken((ref loc, ..))