From a72ef1e3659fa5f429c59153d731c2ee32532dc4 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 27 Jul 2019 13:31:13 -0500 Subject: [PATCH 1/2] Add frozen package support --- bytecode/src/bytecode.rs | 5 +++ derive/src/compile_bytecode.rs | 66 ++++++++++++++++++++++++---------- vm/src/frozen.rs | 4 +-- vm/src/import.rs | 2 +- vm/src/stdlib/imp.rs | 13 ++++--- vm/src/vm.rs | 2 +- 6 files changed, 66 insertions(+), 26 deletions(-) diff --git a/bytecode/src/bytecode.rs b/bytecode/src/bytecode.rs index c6fd093c9e..40da926f41 100644 --- a/bytecode/src/bytecode.rs +++ b/bytecode/src/bytecode.rs @@ -476,3 +476,8 @@ impl fmt::Debug for CodeObject { ) } } + +pub struct FrozenModule { + pub code: CodeObject, + pub package: bool, +} diff --git a/derive/src/compile_bytecode.rs b/derive/src/compile_bytecode.rs index bc61c6e1f2..cfa87163e8 100644 --- a/derive/src/compile_bytecode.rs +++ b/derive/src/compile_bytecode.rs @@ -17,7 +17,7 @@ use crate::{extract_spans, Diagnostic}; use bincode; use proc_macro2::{Span, TokenStream as TokenStream2}; use quote::quote; -use rustpython_bytecode::bytecode::CodeObject; +use rustpython_bytecode::bytecode::{CodeObject, FrozenModule}; use rustpython_compiler::compile; use std::collections::HashMap; use std::env; @@ -52,7 +52,7 @@ impl CompilationSource { &self, mode: &compile::Mode, module_name: String, - ) -> Result, Diagnostic> { + ) -> Result, Diagnostic> { Ok(match &self.kind { CompilationSourceKind::File(rel_path) => { let mut path = PathBuf::from( @@ -65,10 +65,20 @@ impl CompilationSource { format!("Error reading file {:?}: {}", path, err), ) })?; - hashmap! {module_name.clone() => self.compile_string(&source, mode, module_name.clone())?} + hashmap! { + module_name.clone() => FrozenModule { + code: self.compile_string(&source, mode, module_name.clone())?, + package: false, + }, + } } CompilationSourceKind::SourceCode(code) => { - hashmap! {module_name.clone() => self.compile_string(code, mode, module_name.clone())?} + hashmap! { + module_name.clone() => FrozenModule { + code: self.compile_string(code, mode, module_name.clone())?, + package: false, + }, + } } CompilationSourceKind::Dir(rel_path) => { let mut path = PathBuf::from( @@ -85,7 +95,7 @@ impl CompilationSource { path: &Path, parent: String, mode: &compile::Mode, - ) -> Result, Diagnostic> { + ) -> Result, Diagnostic> { let mut code_map = HashMap::new(); let paths = fs::read_dir(&path).map_err(|err| { Diagnostic::spans_error(self.span, format!("Error listing dir {:?}: {}", path, err)) @@ -95,11 +105,13 @@ impl CompilationSource { Diagnostic::spans_error(self.span, format!("Failed to list file: {}", err)) })?; let path = path.path(); - let file_name = path.file_name().unwrap().to_str().unwrap(); + let file_name = path.file_name().unwrap().to_str().ok_or_else(|| { + Diagnostic::spans_error(self.span, format!("Invalid UTF-8 in file name {:?}", path)) + })?; if path.is_dir() { code_map.extend(self.compile_dir( &path, - format!("{}{}.", parent, file_name), + format!("{}{}", parent, file_name), mode, )?); } else if file_name.ends_with(".py") { @@ -109,11 +121,21 @@ impl CompilationSource { format!("Error reading file {:?}: {}", path, err), ) })?; - let file_name_splitte: Vec<&str> = file_name.splitn(2, '.').collect(); - let module_name = format!("{}{}", parent, file_name_splitte[0]); + let stem = path.file_stem().unwrap().to_str().unwrap(); + let is_init = stem == "__init__"; + let module_name = if is_init { + parent.clone() + } else if parent.is_empty() { + stem.to_string() + } else { + format!("{}.{}", parent, stem) + }; code_map.insert( module_name.clone(), - self.compile_string(&source, mode, module_name)?, + FrozenModule { + code: self.compile_string(&source, mode, module_name)?, + package: is_init, + }, ); } } @@ -128,7 +150,7 @@ struct PyCompileInput { } impl PyCompileInput { - fn compile(&self) -> Result, Diagnostic> { + fn compile(&self) -> Result, Diagnostic> { let mut module_name = None; let mut mode = None; let mut source: Option = None; @@ -225,13 +247,21 @@ pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result bincode::deserialize::<::rustpython_vm::bytecode::CodeObject>(#bytes) - .expect("Deserializing CodeObject failed") } - }); + let modules = code_map + .into_iter() + .map(|(module_name, FrozenModule { code, package })| { + let module_name = LitStr::new(&module_name, Span::call_site()); + let bytes = bincode::serialize(&code).expect("Failed to serialize"); + let bytes = LitByteStr::new(&bytes, Span::call_site()); + quote! { + #module_name.into() => ::rustpython_vm::bytecode::FrozenModule { + code: bincode::deserialize::<::rustpython_vm::bytecode::CodeObject>( + #bytes + ).expect("Deserializing CodeObject failed"), + package: #package, + } + } + }); let output = quote! { ({ diff --git a/vm/src/frozen.rs b/vm/src/frozen.rs index b9ea11ac13..916327bbc8 100644 --- a/vm/src/frozen.rs +++ b/vm/src/frozen.rs @@ -1,7 +1,7 @@ -use crate::bytecode::CodeObject; +use crate::bytecode::FrozenModule; use std::collections::HashMap; -pub fn get_module_inits() -> HashMap { +pub fn get_module_inits() -> HashMap { let mut modules = HashMap::new(); modules.extend(py_compile_bytecode!( source = "initialized = True; print(\"Hello world!\")\n", diff --git a/vm/src/import.rs b/vm/src/import.rs index ea977c91b9..d48d36466f 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -43,7 +43,7 @@ pub fn import_frozen(vm: &VirtualMachine, module_name: &str) -> PyResult { .borrow() .get(module_name) .ok_or_else(|| vm.new_import_error(format!("Cannot import frozen module {}", module_name))) - .and_then(|frozen| import_codeobj(vm, module_name, frozen.clone(), false)) + .and_then(|frozen| import_codeobj(vm, module_name, frozen.code.clone(), false)) } pub fn import_builtin(vm: &VirtualMachine, module_name: &str) -> PyResult { diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index 6f7ecd1bdb..87f2179f91 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -57,7 +57,7 @@ fn imp_get_frozen_object(name: PyStringRef, vm: &VirtualMachine) -> PyResult PyResult { import::import_frozen(vm, name.as_str()) } -fn imp_is_frozen_package(_name: PyStringRef, _vm: &VirtualMachine) -> bool { - // TODO: Support frozen package. - false +fn imp_is_frozen_package(name: PyStringRef, vm: &VirtualMachine) -> PyResult { + vm.frozen + .borrow() + .get(name.as_str()) + .map(|frozen| frozen.package) + .ok_or_else(|| { + vm.new_import_error(format!("No such frozen object named {}", name.as_str())) + }) } fn imp_fix_co_filename(_code: PyObjectRef, _path: PyStringRef, _vm: &VirtualMachine) { diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 1bf8e21c33..3c308002ed 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -56,7 +56,7 @@ pub struct VirtualMachine { pub frames: RefCell>, pub wasm_id: Option, pub exceptions: RefCell>, - pub frozen: RefCell>, + pub frozen: RefCell>, pub import_func: RefCell, pub profile_func: RefCell, pub trace_func: RefCell, From ec6c016fa56d6c9c7f6f2e3518ceeab5dacec0d4 Mon Sep 17 00:00:00 2001 From: coolreader18 <33094578+coolreader18@users.noreply.github.com> Date: Sat, 27 Jul 2019 13:51:13 -0500 Subject: [PATCH 2/2] Fix wasm errors --- wasm/lib/src/convert.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/wasm/lib/src/convert.rs b/wasm/lib/src/convert.rs index 6de4a5987a..f4c71e60d5 100644 --- a/wasm/lib/src/convert.rs +++ b/wasm/lib/src/convert.rs @@ -22,13 +22,9 @@ pub fn py_err_to_js_err(vm: &VirtualMachine, py_err: &PyObjectRef) -> JsValue { } }; } - let msg = match vm - .get_attribute(py_err.clone(), "msg") - .ok() - .and_then(|msg| vm.to_pystr(&msg).ok()) - { - Some(msg) => msg, - None => return js_sys::Error::new("error getting error").into(), + let msg = match vm.to_pystr(py_err) { + Ok(msg) => msg, + Err(_) => return js_sys::Error::new("error getting error").into(), }; let js_err = map_exceptions!(py_err,& msg, { // TypeError is sort of a catch-all for "this value isn't what I thought it was like"