From 71e8c5e56ee90e08a406ae57b6ded357f26fb5d7 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sun, 15 Mar 2020 21:33:00 -0500 Subject: [PATCH 1/6] Add msvcrt.setmode --- vm/src/stdlib/msvcrt.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/vm/src/stdlib/msvcrt.rs b/vm/src/stdlib/msvcrt.rs index 6861328520..367eff4f14 100644 --- a/vm/src/stdlib/msvcrt.rs +++ b/vm/src/stdlib/msvcrt.rs @@ -1,3 +1,4 @@ +use super::os::errno_err; use crate::obj::objbytes::PyBytesRef; use crate::obj::objstr::PyStringRef; use crate::pyobject::{PyObjectRef, PyResult}; @@ -45,6 +46,19 @@ fn msvcrt_putwch(s: PyStringRef, vm: &VirtualMachine) -> PyResult<()> { Ok(()) } +extern "C" { + fn _setmode(fd: i32, flags: i32) -> i32; +} + +fn msvcrt_setmode(fd: i32, flags: i32, vm: &VirtualMachine) -> PyResult { + let flags = unsafe { suppress_iph!(_setmode(fd, flags)) }; + if flags == -1 { + Err(errno_err(vm)) + } else { + Ok(flags) + } +} + pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let ctx = &vm.ctx; py_module!(vm, "_msvcrt", { @@ -54,5 +68,6 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "getwche" => ctx.new_function(msvcrt_getwche), "putch" => ctx.new_function(msvcrt_putch), "putwch" => ctx.new_function(msvcrt_putwch), + "setmode" => ctx.new_function(msvcrt_setmode), }) } From f87b2d9af991863802e5bf533297ad5d7c16214a Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Mon, 16 Mar 2020 21:28:35 -0500 Subject: [PATCH 2/6] Improve unwrap_pyresult for wasm --- vm/src/vm.rs | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/vm/src/vm.rs b/vm/src/vm.rs index cb93b47b9d..d8d2e7b0c7 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -537,14 +537,30 @@ impl VirtualMachine { // TODO: #[track_caller] when stabilized fn _py_panic_failed(&self, exc: &PyBaseExceptionRef, msg: &str) -> ! { - let show_backtrace = env::var_os("RUST_BACKTRACE").map_or(false, |v| &v != "0"); - let after = if show_backtrace { - exceptions::print_exception(self, exc); - "exception backtrace above" - } else { - "run with RUST_BACKTRACE=1 to see Python backtrace" - }; - panic!("{}; {}", msg, after) + #[cfg(not(all(target_arch = "wasm32", not(target_os = "wasi"))))] + { + let show_backtrace = env::var_os("RUST_BACKTRACE").map_or(false, |v| &v != "0"); + let after = if show_backtrace { + exceptions::print_exception(self, exc); + "exception backtrace above" + } else { + "run with RUST_BACKTRACE=1 to see Python backtrace" + }; + panic!("{}; {}", msg, after) + } + #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] + { + use wasm_bindgen::prelude::*; + #[wasm_bindgen] + extern "C" { + #[wasm_bindgen(js_namespace = console)] + fn error(s: &str); + } + let mut s = Vec::::new(); + exceptions::write_exception(&mut s, self, exc).unwrap(); + error(std::str::from_utf8(&s).unwrap()); + panic!("{}; exception backtrace above", msg) + } } pub fn unwrap_pyresult(&self, result: PyResult) -> T { result.unwrap_or_else(|exc| { From 000ec722f812b93e69ae9e4a34dcdbbe3690ef64 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Mon, 16 Mar 2020 23:18:13 -0500 Subject: [PATCH 3/6] Allow dummy faulthandler functions to accept kwargs --- vm/src/stdlib/faulthandler.rs | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/vm/src/stdlib/faulthandler.rs b/vm/src/stdlib/faulthandler.rs index bbf11fc7ad..e97d1c31d7 100644 --- a/vm/src/stdlib/faulthandler.rs +++ b/vm/src/stdlib/faulthandler.rs @@ -20,16 +20,33 @@ fn dump_traceback(_file: OptionalArg, _all_threads: OptionalArg, vm: } } -fn enable(_file: OptionalArg, _all_threads: OptionalArg) { +#[derive(FromArgs)] +#[allow(unused)] +struct EnableArgs { + #[pyarg(positional_or_keyword, default = "None")] + file: Option, + #[pyarg(positional_or_keyword, default = "true")] + all_threads: bool, +} + +fn enable(_args: EnableArgs) { // TODO } -fn register( - _signum: i64, - _file: OptionalArg, - _all_threads: OptionalArg, - _chain: OptionalArg, -) { +#[derive(FromArgs)] +#[allow(unused)] +struct RegisterArgs { + #[pyarg(positional_only)] + signum: i64, + #[pyarg(positional_or_keyword, default = "None")] + file: Option, + #[pyarg(positional_or_keyword, default = "true")] + all_threads: bool, + #[pyarg(positional_or_keyword, default = "false")] + chain: bool, +} + +fn register(_args: RegisterArgs) { // TODO } From a529fdbc6ef3c926c8e03b4b33e44cccb32a2c41 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Fri, 27 Mar 2020 16:25:09 -0500 Subject: [PATCH 4/6] Fix os.set_inheritable on windows --- vm/src/stdlib/os.rs | 69 +++++++++++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 21 deletions(-) diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 212c1cb224..442c7085e6 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -22,6 +22,8 @@ use nix::pty::openpty; use nix::unistd::{self, Gid, Pid, Uid}; #[cfg(unix)] use std::os::unix::io::RawFd; +#[cfg(windows)] +use std::os::windows::io::RawHandle; #[cfg(unix)] use uname; @@ -70,7 +72,7 @@ pub fn raw_file_number(handle: File) -> i64 { #[cfg(windows)] pub fn rust_file(raw_fileno: i64) -> File { - use std::os::windows::io::{AsRawHandle, FromRawHandle, RawHandle}; + use std::os::windows::io::{AsRawHandle, FromRawHandle}; let raw_fileno = match raw_fileno { 0 => io::stdin().as_raw_handle(), @@ -975,22 +977,42 @@ fn os_get_inheritable(fd: RawFd, vm: &VirtualMachine) -> PyResult { } } -#[cfg(unix)] -fn os_set_inheritable(fd: RawFd, inheritable: bool, vm: &VirtualMachine) -> PyResult<()> { - let _set_flag = || { - use nix::fcntl::fcntl; - use nix::fcntl::FcntlArg; - use nix::fcntl::FdFlag; - - let flags = FdFlag::from_bits_truncate(fcntl(fd, FcntlArg::F_GETFD)?); - let mut new_flags = flags; - new_flags.set(FdFlag::from_bits_truncate(libc::FD_CLOEXEC), !inheritable); - if flags != new_flags { - fcntl(fd, FcntlArg::F_SETFD(new_flags))?; +fn os_set_inheritable(fd: i64, inheritable: bool, vm: &VirtualMachine) -> PyResult<()> { + #[cfg(unix)] + { + let fd = fd as RawFd; + let _set_flag = || { + use nix::fcntl::fcntl; + use nix::fcntl::FcntlArg; + use nix::fcntl::FdFlag; + + let flags = FdFlag::from_bits_truncate(fcntl(fd, FcntlArg::F_GETFD)?); + let mut new_flags = flags; + new_flags.set(FdFlag::from_bits_truncate(libc::FD_CLOEXEC), !inheritable); + if flags != new_flags { + fcntl(fd, FcntlArg::F_SETFD(new_flags))?; + } + Ok(()) + }; + _set_flag().or_else(|err| Err(convert_nix_error(vm, err))) + } + #[cfg(windows)] + { + use winapi::um::{handleapi, winbase}; + let fd = fd as RawHandle; + let flags = if inheritable { + winbase::HANDLE_FLAG_INHERIT + } else { + 0 + }; + let ret = + unsafe { handleapi::SetHandleInformation(fd, winbase::HANDLE_FLAG_INHERIT, flags) }; + if ret == 0 { + Err(errno_err(vm)) + } else { + Ok(()) } - Ok(()) - }; - _set_flag().or_else(|err| Err(convert_nix_error(vm, err))) + } } #[cfg(unix)] @@ -1027,8 +1049,8 @@ fn os_pipe(vm: &VirtualMachine) -> PyResult<(RawFd, RawFd)> { use nix::unistd::close; use nix::unistd::pipe; let (rfd, wfd) = pipe().map_err(|err| convert_nix_error(vm, err))?; - os_set_inheritable(rfd, false, vm) - .and_then(|_| os_set_inheritable(wfd, false, vm)) + os_set_inheritable(rfd.into(), false, vm) + .and_then(|_| os_set_inheritable(wfd.into(), false, vm)) .or_else(|err| { let _ = close(rfd); let _ = close(wfd); @@ -1439,6 +1461,7 @@ pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { "urandom" => ctx.new_function(os_urandom), "isatty" => ctx.new_function(os_isatty), "lseek" => ctx.new_function(os_lseek), + "set_inheritable" => ctx.new_function(os_set_inheritable), "O_RDONLY" => ctx.new_int(libc::O_RDONLY), "O_WRONLY" => ctx.new_int(libc::O_WRONLY), @@ -1506,7 +1529,6 @@ fn extend_module_platform_specific(vm: &VirtualMachine, module: PyObjectRef) -> "getuid" => ctx.new_function(os_getuid), "geteuid" => ctx.new_function(os_geteuid), "pipe" => ctx.new_function(os_pipe), //TODO: windows - "set_inheritable" => ctx.new_function(os_set_inheritable), // TODO: windows "set_blocking" => ctx.new_function(os_set_blocking), "setgid" => ctx.new_function(os_setgid), "setpgid" => ctx.new_function(os_setpgid), @@ -1578,7 +1600,12 @@ fn extend_module_platform_specific(vm: &VirtualMachine, module: PyObjectRef) -> module } -#[cfg(not(unix))] -fn extend_module_platform_specific(_vm: &VirtualMachine, module: PyObjectRef) -> PyObjectRef { +#[cfg(windows)] +fn extend_module_platform_specific(vm: &VirtualMachine, module: PyObjectRef) -> PyObjectRef { + let ctx = &vm.ctx; + extend_module!(vm, module, { + "O_BINARY" => ctx.new_int(libc::O_BINARY), + }); + module } From 942a7d612c9b1d1ae8018568fb1489fba9337930 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Sat, 28 Mar 2020 00:16:27 -0500 Subject: [PATCH 5/6] Fix os.lseek on Windows --- vm/Cargo.toml | 2 +- vm/src/stdlib/os.rs | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/vm/Cargo.toml b/vm/Cargo.toml index 840302fa98..5dd3bb6c52 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -108,7 +108,7 @@ schannel = "0.1" [target."cfg(windows)".dependencies.winapi] version = "0.3" -features = ["winsock2", "handleapi", "ws2def", "std", "winbase", "wincrypt"] +features = ["winsock2", "handleapi", "ws2def", "std", "winbase", "wincrypt", "fileapi"] [target.'cfg(target_arch = "wasm32")'.dependencies] wasm-bindgen = "0.2" diff --git a/vm/src/stdlib/os.rs b/vm/src/stdlib/os.rs index 442c7085e6..0224c0e40c 100644 --- a/vm/src/stdlib/os.rs +++ b/vm/src/stdlib/os.rs @@ -1344,10 +1344,25 @@ fn os_isatty(fd: i32) -> bool { fn os_lseek(fd: i32, position: Offset, how: i32, vm: &VirtualMachine) -> PyResult { #[cfg(not(windows))] - use libc::lseek; + let res = unsafe { suppress_iph!(libc::lseek(fd, position, how)) }; #[cfg(windows)] - use libc::lseek64 as lseek; - let res = unsafe { suppress_iph!(lseek(fd, position, how)) }; + let res = unsafe { + use winapi::um::{fileapi, winnt}; + let mut li = winnt::LARGE_INTEGER::default(); + *li.QuadPart_mut() = position; + let ret = fileapi::SetFilePointer( + fd as RawHandle, + li.u().LowPart as _, + &mut li.u_mut().HighPart, + how as _, + ); + if ret == fileapi::INVALID_SET_FILE_POINTER { + -1 + } else { + li.u_mut().LowPart = ret; + *li.QuadPart() + } + }; if res < 0 { Err(errno_err(vm)) } else { From a540f042e8ddcf52b385a476b6adef1c997bf609 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Fri, 27 Mar 2020 17:10:09 -0500 Subject: [PATCH 6/6] Add _codecs.charmap_build, fix other _codecs issues --- Lib/_codecs.py | 9 ++++++--- Lib/test/test_unicode.py | 2 -- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Lib/_codecs.py b/Lib/_codecs.py index 26439b0614..78da31d3f1 100644 --- a/Lib/_codecs.py +++ b/Lib/_codecs.py @@ -239,6 +239,9 @@ def charmap_encode(obj, errors='strict', mapping='latin-1'): res = bytes(res) return res, len(res) +def charmap_build(s): + return {ord(c): i for i, c in enumerate(s)} + if sys.maxunicode == 65535: unicode_bytes = 2 else: @@ -1525,11 +1528,11 @@ def charmapencode_output(c, mapping): rep = mapping[c] if isinstance(rep, int) or isinstance(rep, int): if rep < 256: - return chr(rep) + return rep else: raise TypeError("character mapping must be in range(256)") elif isinstance(rep, str): - return rep + return ord(rep) elif rep == None: raise KeyError("character maps to ") else: @@ -1579,7 +1582,7 @@ def PyUnicode_DecodeCharmap(s, size, mapping, errors): #/* Get mapping (char ordinal -> integer, Unicode char or None) */ ch = s[inpos] try: - x = mapping[ord(ch)] + x = mapping[ch] if isinstance(x, int): if x < 65536: p += chr(x) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 476540e215..88843dedc1 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2219,8 +2219,6 @@ def test_codecs(self): for encoding in ('utf-8',): self.assertEqual(str(u.encode(encoding),encoding), u) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_codecs_charmap(self): # 0-127 s = bytes(range(128))