From befe1803dba0cfff17ec372027fe500d686fd410 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 20 Jul 2019 12:23:16 +0300 Subject: [PATCH 1/7] Add freeze-stdlib compile flag --- Cargo.toml | 1 + vm/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 0fef0d99ea..9a163b07a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ path = "./benchmarks/bench.rs" [features] default = ["rustpython-vm/use-proc-macro-hack"] flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"] +freeze-stdlib = ["rustpython-vm/freeze-stdlib"] [dependencies] log="0.4.1" diff --git a/vm/Cargo.toml b/vm/Cargo.toml index ec27b0434d..0b15c0b2eb 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -13,6 +13,7 @@ default = ["rustpython-parser", "rustpython-compiler", "use-proc-macro-hack"] vm-tracing-logging = [] flame-it = ["flame", "flamer"] use-proc-macro-hack = ["proc-macro-hack", "rustpython-derive/proc-macro-hack"] +freeze-stdlib = [] [dependencies] # Crypto: From f271a0e5cf6256fe4b2455752349479dae325acb Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 20 Jul 2019 12:23:56 +0300 Subject: [PATCH 2/7] Change py_compile_bytecode to return map --- derive/src/compile_bytecode.rs | 8 +++++--- vm/src/frozen.rs | 29 +++++++++++++++-------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/derive/src/compile_bytecode.rs b/derive/src/compile_bytecode.rs index f03b5d065d..75d51d108b 100644 --- a/derive/src/compile_bytecode.rs +++ b/derive/src/compile_bytecode.rs @@ -23,7 +23,7 @@ use std::env; use std::fs; use std::path::PathBuf; use syn::parse::{Parse, ParseStream, Result as ParseResult}; -use syn::{self, parse2, Lit, LitByteStr, Meta, Token}; +use syn::{self, parse2, Lit, LitByteStr, LitStr, Meta, Token}; enum CompilationSourceKind { File(PathBuf), @@ -156,14 +156,16 @@ pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result(#bytes) - .expect("Deserializing CodeObject failed") + hashmap! { #module_name.into() => bincode::deserialize::<::rustpython_vm::bytecode::CodeObject>(#bytes) + .expect("Deserializing CodeObject failed")} }) }; diff --git a/vm/src/frozen.rs b/vm/src/frozen.rs index 5b9ffe1992..19a8e26539 100644 --- a/vm/src/frozen.rs +++ b/vm/src/frozen.rs @@ -2,18 +2,19 @@ use crate::bytecode::CodeObject; use std::collections::HashMap; pub fn get_module_inits() -> HashMap { - hashmap! { - "__hello__".into() => py_compile_bytecode!( - source = "initialized = True; print(\"Hello world!\")\n", - module_name = "__hello__", - ), - "_frozen_importlib".into() => py_compile_bytecode!( - file = "Lib/_bootstrap.py", - module_name = "_frozen_importlib", - ), - "_frozen_importlib_external".into() => py_compile_bytecode!( - file = "Lib/_bootstrap_external.py", - module_name = "_frozen_importlib_external", - ), - } + let mut modules = HashMap::new(); + modules.extend(py_compile_bytecode!( + source = "initialized = True; print(\"Hello world!\")\n", + module_name = "__hello__", + )); + modules.extend(py_compile_bytecode!( + file = "Lib/_bootstrap.py", + module_name = "_frozen_importlib", + )); + modules.extend(py_compile_bytecode!( + file = "Lib/_bootstrap_external.py", + module_name = "_frozen_importlib_external", + )); + + modules } From 5c240d71a885ae6848a0026997be6d8f276e6ac3 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 20 Jul 2019 13:03:59 +0300 Subject: [PATCH 3/7] Support freezing of stdlib --- Cargo.lock | 1 + derive/Cargo.toml | 1 + derive/src/compile_bytecode.rs | 111 +++++++++++++++++++++++++++------ derive/src/lib.rs | 3 + vm/src/frozen.rs | 5 ++ 5 files changed, 103 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index baf7b521a7..f71f35a718 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -971,6 +971,7 @@ name = "rustpython-derive" version = "0.1.0" dependencies = [ "bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "maplit 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro-hack 0.5.8 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/derive/Cargo.toml b/derive/Cargo.toml index 7bdeb3e44a..429d079c8d 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -21,3 +21,4 @@ rustpython-compiler = { path = "../compiler", version = "0.1.0" } rustpython-bytecode = { path = "../bytecode", version = "0.1.0" } bincode = "1.1" proc-macro-hack = { version = "0.5", optional = true } +maplit = "1.0" diff --git a/derive/src/compile_bytecode.rs b/derive/src/compile_bytecode.rs index 75d51d108b..d994284a1a 100644 --- a/derive/src/compile_bytecode.rs +++ b/derive/src/compile_bytecode.rs @@ -19,6 +19,7 @@ use proc_macro2::{Span, TokenStream as TokenStream2}; use quote::quote; use rustpython_bytecode::bytecode::CodeObject; use rustpython_compiler::compile; +use std::collections::HashMap; use std::env; use std::fs; use std::path::PathBuf; @@ -28,6 +29,7 @@ use syn::{self, parse2, Lit, LitByteStr, LitStr, Meta, Token}; enum CompilationSourceKind { File(PathBuf), SourceCode(String), + Dir(PathBuf), } struct CompilationSource { @@ -36,14 +38,22 @@ struct CompilationSource { } impl CompilationSource { - fn compile(self, mode: &compile::Mode, module_name: String) -> Result { - let compile = |source| { - compile::compile(source, mode, module_name, 0).map_err(|err| { - Diagnostic::spans_error(self.span, format!("Compile error: {}", err)) - }) - }; - - match &self.kind { + fn compile_string( + &self, + source: &str, + mode: &compile::Mode, + module_name: String, + ) -> Result { + compile::compile(source, mode, module_name, 0) + .map_err(|err| Diagnostic::spans_error(self.span, format!("Compile error: {}", err))) + } + + fn compile( + &self, + mode: &compile::Mode, + module_name: String, + ) -> Result, Diagnostic> { + Ok(match &self.kind { CompilationSourceKind::File(rel_path) => { let mut path = PathBuf::from( env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not present"), @@ -55,10 +65,61 @@ impl CompilationSource { format!("Error reading file {:?}: {}", path, err), ) })?; - compile(&source) + hashmap! {module_name.clone() => self.compile_string(&source, mode, module_name.clone())?} + } + CompilationSourceKind::SourceCode(code) => { + hashmap! {module_name.clone() => self.compile_string(code, mode, module_name.clone())?} + } + CompilationSourceKind::Dir(rel_path) => { + let mut path = PathBuf::from( + env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not present"), + ); + path.push(rel_path); + self.compile_dir(&path, "".to_string(), mode)? + } + }) + } + + fn compile_dir( + &self, + path: &PathBuf, + parent: String, + mode: &compile::Mode, + ) -> Result, Diagnostic> { + let mut code_map = HashMap::new(); + let paths = fs::read_dir(&path).map_err(|err| { + Diagnostic::spans_error(self.span, format!("Error listing dir {:?}: {}", path, err)) + })?; + for path in paths { + let path = path.map_err(|err| { + Diagnostic::spans_error(self.span, format!("Failed to list file: {}", err)) + })?; + let path = path.path(); + let file_name = path.file_name().unwrap().to_str().unwrap(); + if path.is_dir() { + code_map.extend(self.compile_dir( + &path, + format!("{}{}.", parent, file_name), + mode, + )?); + } else { + if file_name.ends_with(".py") { + let source = fs::read_to_string(&path).map_err(|err| { + Diagnostic::spans_error( + self.span, + format!("Error reading file {:?}: {}", path, err), + ) + })?; + let file_name_splitte: Vec<&str> = file_name.splitn(2, ".").collect(); + let module_name = format!("{}{}", parent, file_name_splitte[0]); + code_map.insert( + module_name.clone(), + self.compile_string(&source, mode, module_name)?, + ); + } } - CompilationSourceKind::SourceCode(code) => compile(code), } + Ok(code_map) } } @@ -69,7 +130,7 @@ struct PyCompileInput { } impl PyCompileInput { - fn compile(&self) -> Result { + fn compile(&self) -> Result, Diagnostic> { let mut module_name = None; let mut mode = None; let mut source: Option = None; @@ -122,6 +183,16 @@ impl PyCompileInput { kind: CompilationSourceKind::File(path), span: extract_spans(&name_value).unwrap(), }); + } else if name_value.ident == "dir" { + assert_source_empty(&source)?; + let path = match &name_value.lit { + Lit::Str(s) => PathBuf::from(s.value()), + _ => bail_span!(name_value.lit, "source must be a string"), + }; + source = Some(CompilationSource { + kind: CompilationSourceKind::Dir(path), + span: extract_spans(&name_value).unwrap(), + }); } } } @@ -154,18 +225,22 @@ impl Parse for PyCompileInput { pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result { let input: PyCompileInput = parse2(input)?; - let code_obj = input.compile()?; - - let module_name = LitStr::new(&code_obj.source_path, Span::call_site()); + let code_map = input.compile()?; - let bytes = bincode::serialize(&code_obj).expect("Failed to serialize"); - let bytes = LitByteStr::new(&bytes, Span::call_site()); + let modules = code_map.iter().map(|(module_name, code_obj)| { + let module_name = LitStr::new(&module_name, Span::call_site()); + let bytes = bincode::serialize(&code_obj).expect("Failed to serialize"); + let bytes = LitByteStr::new(&bytes, Span::call_site()); + quote! { #module_name.into() => bincode::deserialize::<::rustpython_vm::bytecode::CodeObject>(#bytes) + .expect("Deserializing CodeObject failed") } + }); let output = quote! { ({ use ::rustpython_vm::__exports::bincode; - hashmap! { #module_name.into() => bincode::deserialize::<::rustpython_vm::bytecode::CodeObject>(#bytes) - .expect("Deserializing CodeObject failed")} + hashmap! { + #(#modules),* + } }) }; diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 33e075d305..725e763a5e 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -4,6 +4,9 @@ extern crate proc_macro; +#[macro_use] +extern crate maplit; + #[macro_use] mod error; mod compile_bytecode; diff --git a/vm/src/frozen.rs b/vm/src/frozen.rs index 19a8e26539..b9ea11ac13 100644 --- a/vm/src/frozen.rs +++ b/vm/src/frozen.rs @@ -16,5 +16,10 @@ pub fn get_module_inits() -> HashMap { module_name = "_frozen_importlib_external", )); + #[cfg(feature = "freeze-stdlib")] + { + modules.extend(py_compile_bytecode!(dir = "../Lib/",)); + } + modules } From c48044942069b643bca921c15cfd07ca5e9f029b Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 20 Jul 2019 22:09:24 +0300 Subject: [PATCH 4/7] Fix clippy comments --- derive/src/compile_bytecode.rs | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/derive/src/compile_bytecode.rs b/derive/src/compile_bytecode.rs index d994284a1a..896d8f2a15 100644 --- a/derive/src/compile_bytecode.rs +++ b/derive/src/compile_bytecode.rs @@ -75,7 +75,7 @@ impl CompilationSource { env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR is not present"), ); path.push(rel_path); - self.compile_dir(&path, "".to_string(), mode)? + self.compile_dir(&path, String::new(), mode)? } }) } @@ -102,21 +102,19 @@ impl CompilationSource { format!("{}{}.", parent, file_name), mode, )?); - } else { - if file_name.ends_with(".py") { - let source = fs::read_to_string(&path).map_err(|err| { - Diagnostic::spans_error( - self.span, - format!("Error reading file {:?}: {}", path, err), - ) - })?; - let file_name_splitte: Vec<&str> = file_name.splitn(2, ".").collect(); - let module_name = format!("{}{}", parent, file_name_splitte[0]); - code_map.insert( - module_name.clone(), - self.compile_string(&source, mode, module_name)?, - ); - } + } else if file_name.ends_with(".py") { + let source = fs::read_to_string(&path).map_err(|err| { + Diagnostic::spans_error( + self.span, + format!("Error reading file {:?}: {}", path, err), + ) + })?; + let file_name_splitte: Vec<&str> = file_name.splitn(2, ".").collect(); + let module_name = format!("{}{}", parent, file_name_splitte[0]); + code_map.insert( + module_name.clone(), + self.compile_string(&source, mode, module_name)?, + ); } } Ok(code_map) From 7226c925d2801efb1b310eb5683c8179f4b005c0 Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Sat, 20 Jul 2019 22:25:46 +0300 Subject: [PATCH 5/7] Add hashmap macro to __exports --- derive/src/compile_bytecode.rs | 1 + vm/src/lib.rs | 1 + wasm/lib/src/browser_module.rs | 8 ++++---- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/derive/src/compile_bytecode.rs b/derive/src/compile_bytecode.rs index 896d8f2a15..458d151765 100644 --- a/derive/src/compile_bytecode.rs +++ b/derive/src/compile_bytecode.rs @@ -236,6 +236,7 @@ pub fn impl_py_compile_bytecode(input: TokenStream2) -> Result Date: Mon, 22 Jul 2019 17:58:33 +0300 Subject: [PATCH 6/7] Fix clippy warning --- derive/src/compile_bytecode.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/derive/src/compile_bytecode.rs b/derive/src/compile_bytecode.rs index 458d151765..7ffc956abc 100644 --- a/derive/src/compile_bytecode.rs +++ b/derive/src/compile_bytecode.rs @@ -109,7 +109,7 @@ impl CompilationSource { format!("Error reading file {:?}: {}", path, err), ) })?; - let file_name_splitte: Vec<&str> = file_name.splitn(2, ".").collect(); + let file_name_splitte: Vec<&str> = file_name.splitn(2, '.').collect(); let module_name = format!("{}{}", parent, file_name_splitte[0]); code_map.insert( module_name.clone(), From 0a6253593f73cadd709890c02cd0c54c952e456a Mon Sep 17 00:00:00 2001 From: Aviv Palivoda Date: Mon, 22 Jul 2019 18:02:57 +0300 Subject: [PATCH 7/7] Change compile_dir path to &Path --- derive/src/compile_bytecode.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/derive/src/compile_bytecode.rs b/derive/src/compile_bytecode.rs index 7ffc956abc..bc61c6e1f2 100644 --- a/derive/src/compile_bytecode.rs +++ b/derive/src/compile_bytecode.rs @@ -22,7 +22,7 @@ use rustpython_compiler::compile; use std::collections::HashMap; use std::env; use std::fs; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use syn::parse::{Parse, ParseStream, Result as ParseResult}; use syn::{self, parse2, Lit, LitByteStr, LitStr, Meta, Token}; @@ -82,7 +82,7 @@ impl CompilationSource { fn compile_dir( &self, - path: &PathBuf, + path: &Path, parent: String, mode: &compile::Mode, ) -> Result, Diagnostic> {