From 85030e368a37abfd14da707e6360eb6fc48f512c Mon Sep 17 00:00:00 2001 From: Noa Date: Fri, 14 Oct 2022 16:50:37 -0500 Subject: [PATCH 1/4] Make rustpython-vm compatible with non-js wasm32-unknown & add tests --- .github/workflows/ci.yaml | 8 ++++++++ Cargo.lock | 8 ++++++++ Cargo.toml | 3 ++- stdlib/Cargo.toml | 2 +- vm/Cargo.toml | 10 ++++++---- vm/src/stdlib/time.rs | 7 ++++++- vm/src/vm/vm_object.rs | 7 ++++++- wasm/lib/Cargo.toml | 2 +- wasm/wasm-unknown-test/Cargo.toml | 11 +++++++++++ wasm/wasm-unknown-test/README.md | 1 + wasm/wasm-unknown-test/src/lib.rs | 16 ++++++++++++++++ 11 files changed, 66 insertions(+), 9 deletions(-) create mode 100644 wasm/wasm-unknown-test/Cargo.toml create mode 100644 wasm/wasm-unknown-test/README.md create mode 100644 wasm/wasm-unknown-test/src/lib.rs diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bec7eafa32..c08bb3a991 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -380,6 +380,14 @@ jobs: env: NODE_OPTIONS: "--openssl-legacy-provider" working-directory: ./wasm/demo + - uses: mwilliamson/setup-wabt-action@v1 + with: { wabt-version: "1.0.30" } + - name: check wasm32-unknown without js + run: | + cargo build --release -p wasm-unknown-test --target wasm32-unknown-unknown --verbose + if wasm-objdump -xj Import target/wasm32-unknown-unknown/release/wasm_unknown_test.wasm; then + echo "ERROR: wasm32-unknown module expects imports from the host environment" >2 + fi - name: build notebook demo if: github.ref == 'refs/heads/release' run: | diff --git a/Cargo.lock b/Cargo.lock index 5fcb773d84..b5d99cb2a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3154,6 +3154,14 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +[[package]] +name = "wasm-unknown-test" +version = "0.1.0" +dependencies = [ + "getrandom", + "rustpython-vm", +] + [[package]] name = "web-sys" version = "0.3.61" diff --git a/Cargo.toml b/Cargo.toml index eec5d84a8f..741aab8856 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -96,7 +96,8 @@ x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies = resolver = "2" members = [ "compiler", "compiler/core", "compiler/codegen", - ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "wasm/lib", "derive-impl", + ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl", + "wasm/lib", "wasm/wasm-unknown-test", ] [workspace.package] diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 485855d646..1e327c4e9d 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -22,7 +22,7 @@ ssl-vendor = ["ssl", "openssl/vendored", "openssl-probe"] [dependencies] # rustpython crates rustpython-derive = { workspace = true } -rustpython-vm = { workspace = true } +rustpython-vm = { workspace = true, default-features = false } rustpython-common = { workspace = true } ahash = { workspace = true } diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 251d482ebe..7e96a983d9 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -10,7 +10,7 @@ repository.workspace = true license.workspace = true [features] -default = ["compiler"] +default = ["compiler", "js"] importlib = [] encodings = ["importlib"] vm-tracing-logging = [] @@ -24,6 +24,8 @@ codegen = ["rustpython-codegen", "ast"] parser = ["rustpython-parser", "ast"] serde = ["dep:serde"] +js = ["getrandom/js", "chrono/wasmbind", "wasm-bindgen"] + [dependencies] rustpython-compiler = { workspace = true, optional = true } rustpython-codegen = { workspace = true, optional = true } @@ -71,7 +73,7 @@ thread_local = { workspace = true } memchr = { workspace = true } caseless = "0.2.1" -getrandom = { version = "0.2.12", features = ["js"] } +getrandom = { version = "0.2.12" } flamer = { version = "0.4", optional = true } half = "1.8.2" memoffset = "0.9.1" @@ -140,8 +142,8 @@ features = [ "Win32_UI_WindowsAndMessaging", ] -[target.'cfg(target_arch = "wasm32")'.dependencies] -wasm-bindgen = "0.2.92" +[target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] +wasm-bindgen = { version = "0.2.92", optional = true } [build-dependencies] glob = { workspace = true } diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index d026f2953e..ac98335b37 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -102,7 +102,7 @@ mod decl { Ok(duration_since_system_now(vm)?.as_secs_f64()) } - #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] + #[cfg(all(target_arch = "wasm32", feature = "js", not(target_os = "wasi")))] fn _time(_vm: &VirtualMachine) -> PyResult { use wasm_bindgen::prelude::*; #[wasm_bindgen] @@ -115,6 +115,11 @@ mod decl { Ok(Date::now() / 1000.0) } + #[cfg(all(target_arch = "wasm32", not(feature = "js"), not(target_os = "wasi")))] + fn _time(vm: &VirtualMachine) -> PyResult { + Err(vm.new_not_implemented_error("time.time".to_owned())) + } + #[pyfunction] fn monotonic(vm: &VirtualMachine) -> PyResult { Ok(get_monotonic_time(vm)?.as_secs_f64()) diff --git a/vm/src/vm/vm_object.rs b/vm/src/vm/vm_object.rs index e3d1436882..c441e4de4e 100644 --- a/vm/src/vm/vm_object.rs +++ b/vm/src/vm/vm_object.rs @@ -19,7 +19,7 @@ impl VirtualMachine { self.flush_std(); panic!("{msg}") } - #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] + #[cfg(all(target_arch = "wasm32", feature = "js", not(target_os = "wasi")))] { use wasm_bindgen::prelude::*; #[wasm_bindgen] @@ -32,6 +32,11 @@ impl VirtualMachine { error(&s); panic!("{}; exception backtrace above", msg) } + #[cfg(all(target_arch = "wasm32", not(feature = "js"), not(target_os = "wasi")))] + { + let _ = exc; + panic!("{}; python exception not available", msg) + } } pub(crate) fn flush_std(&self) { diff --git a/wasm/lib/Cargo.toml b/wasm/lib/Cargo.toml index 9b71bbce25..200a9ef1e9 100644 --- a/wasm/lib/Cargo.toml +++ b/wasm/lib/Cargo.toml @@ -21,7 +21,7 @@ rustpython-common = { workspace = true } rustpython-pylib = { workspace = true, optional = true } rustpython-stdlib = { workspace = true, default-features = false, optional = true } # make sure no threading! otherwise wasm build will fail -rustpython-vm = { workspace = true, features = ["compiler", "encodings", "serde"] } +rustpython-vm = { workspace = true, features = ["compiler", "encodings", "serde", "js"] } rustpython-parser = { workspace = true } diff --git a/wasm/wasm-unknown-test/Cargo.toml b/wasm/wasm-unknown-test/Cargo.toml new file mode 100644 index 0000000000..b559016923 --- /dev/null +++ b/wasm/wasm-unknown-test/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "wasm-unknown-test" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +getrandom = { version = "0.2.7", features = ["custom"] } +rustpython-vm = { path = "../../vm", default-features = false, features = ["compiler"] } diff --git a/wasm/wasm-unknown-test/README.md b/wasm/wasm-unknown-test/README.md new file mode 100644 index 0000000000..3c4abd59cf --- /dev/null +++ b/wasm/wasm-unknown-test/README.md @@ -0,0 +1 @@ +A test crate to ensure that `rustpython-vm` compiles on `wasm32-unknown-unknown` without a JS host. diff --git a/wasm/wasm-unknown-test/src/lib.rs b/wasm/wasm-unknown-test/src/lib.rs new file mode 100644 index 0000000000..fd043aea3a --- /dev/null +++ b/wasm/wasm-unknown-test/src/lib.rs @@ -0,0 +1,16 @@ +use rustpython_vm::{eval, Interpreter}; + +pub unsafe extern "C" fn eval(s: *const u8, l: usize) -> u32 { + let src = std::slice::from_raw_parts(s, l); + let src = std::str::from_utf8(src).unwrap(); + Interpreter::without_stdlib(Default::default()).enter(|vm| { + let res = eval::eval(vm, src, vm.new_scope_with_builtins(), "").unwrap(); + res.try_into_value(vm).unwrap() + }) +} + +fn getrandom_always_fail(_buf: &mut [u8]) -> Result<(), getrandom::Error> { + Err(getrandom::Error::UNSUPPORTED) +} + +getrandom::register_custom_getrandom!(getrandom_always_fail); From dcd18ae463e10a9b1b75a655c47cb2508626b515 Mon Sep 17 00:00:00 2001 From: Noa Date: Fri, 14 Oct 2022 17:23:32 -0500 Subject: [PATCH 2/4] Fix compilation without compiler feature --- stdlib/Cargo.toml | 2 +- stdlib/src/dis.rs | 25 +++++++++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 1e327c4e9d..051fe86780 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -12,12 +12,12 @@ license.workspace = true [features] default = ["compiler"] -compiler = ["rustpython-vm/compiler"] threading = ["rustpython-common/threading", "rustpython-vm/threading"] zlib = ["libz-sys", "flate2/zlib"] bz2 = ["bzip2"] ssl = ["openssl", "openssl-sys", "foreign-types-shared"] ssl-vendor = ["ssl", "openssl/vendored", "openssl-probe"] +compiler = ["rustpython-vm/compiler"] [dependencies] # rustpython crates diff --git a/stdlib/src/dis.rs b/stdlib/src/dis.rs index d9322d4871..12c2ea75df 100644 --- a/stdlib/src/dis.rs +++ b/stdlib/src/dis.rs @@ -13,18 +13,23 @@ mod decl { let co = if let Ok(co) = obj.get_attr("__code__", vm) { // Method or function: PyRef::try_from_object(vm, co)? - } else if let Ok(_co_str) = PyStrRef::try_from_object(vm, obj.clone()) { + } else if let Ok(co_str) = PyStrRef::try_from_object(vm, obj.clone()) { #[cfg(not(feature = "compiler"))] - return Err(vm.new_runtime_error( - "dis.dis() with str argument requires `compiler` feature".to_owned(), - )); + { + let _ = co_str; + return Err(vm.new_runtime_error( + "dis.dis() with str argument requires `compiler` feature".to_owned(), + )); + } #[cfg(feature = "compiler")] - vm.compile( - _co_str.as_str(), - crate::vm::compiler::Mode::Exec, - "".to_owned(), - ) - .map_err(|err| vm.new_syntax_error(&err, Some(_co_str.as_str())))? + { + vm.compile( + co_str.as_str(), + crate::vm::compiler::Mode::Exec, + "".to_owned(), + ) + .map_err(|err| vm.new_syntax_error(&err, Some(co_str.as_str())))? + } } else { PyRef::try_from_object(vm, obj)? }; From 4daf8571c2dba0e2bf7af86552afaf84cff4e6f5 Mon Sep 17 00:00:00 2001 From: Noa Date: Sat, 15 Oct 2022 13:03:40 -0500 Subject: [PATCH 3/4] fix CI --- .github/workflows/ci.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c08bb3a991..2d2ec40d4b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -216,13 +216,6 @@ jobs: - name: Check compilation for freebsd run: cargo check --target x86_64-unknown-freebsd - - uses: dtolnay/rust-toolchain@stable - with: - target: wasm32-unknown-unknown - - - name: Check compilation for wasm32 - run: cargo check --target wasm32-unknown-unknown --no-default-features - - uses: dtolnay/rust-toolchain@stable with: target: x86_64-unknown-freebsd From 17abc0f3c163e7d3015094bdc97c3db7a8981d9b Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Fri, 9 Aug 2024 01:28:29 +0900 Subject: [PATCH 4/4] separate wasm-unknown-test --- .github/workflows/ci.yaml | 2 +- Cargo.lock | 8 -------- Cargo.toml | 2 +- wasm/wasm-unknown-test/Cargo.toml | 4 +++- 4 files changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2d2ec40d4b..0f39ed8343 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -377,7 +377,7 @@ jobs: with: { wabt-version: "1.0.30" } - name: check wasm32-unknown without js run: | - cargo build --release -p wasm-unknown-test --target wasm32-unknown-unknown --verbose + cargo build --release --manifest-path wasm/wasm-unknown-test/Cargo.toml --target wasm32-unknown-unknown --verbose if wasm-objdump -xj Import target/wasm32-unknown-unknown/release/wasm_unknown_test.wasm; then echo "ERROR: wasm32-unknown module expects imports from the host environment" >2 fi diff --git a/Cargo.lock b/Cargo.lock index b5d99cb2a3..5fcb773d84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3154,14 +3154,6 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" -[[package]] -name = "wasm-unknown-test" -version = "0.1.0" -dependencies = [ - "getrandom", - "rustpython-vm", -] - [[package]] name = "web-sys" version = "0.3.61" diff --git a/Cargo.toml b/Cargo.toml index 741aab8856..4cc21ad5f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,7 +97,7 @@ resolver = "2" members = [ "compiler", "compiler/core", "compiler/codegen", ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl", - "wasm/lib", "wasm/wasm-unknown-test", + "wasm/lib", ] [workspace.package] diff --git a/wasm/wasm-unknown-test/Cargo.toml b/wasm/wasm-unknown-test/Cargo.toml index b559016923..f5e0b55786 100644 --- a/wasm/wasm-unknown-test/Cargo.toml +++ b/wasm/wasm-unknown-test/Cargo.toml @@ -7,5 +7,7 @@ edition = "2021" crate-type = ["cdylib"] [dependencies] -getrandom = { version = "0.2.7", features = ["custom"] } +getrandom = { version = "0.2.12", features = ["custom"] } rustpython-vm = { path = "../../vm", default-features = false, features = ["compiler"] } + +[workspace]