From 7fd2f0b5e756a3ab9ef38b36bbb22bd91de6b393 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 22 Sep 2021 02:57:37 +0900 Subject: [PATCH 1/8] expose more common interfaces --- vm/src/builtins/memory.rs | 2 +- vm/src/builtins/pystr.rs | 2 +- vm/src/lib.rs | 6 +++--- vm/src/sliceable.rs | 2 +- vm/src/stdlib/io.rs | 2 +- vm/src/stdlib/posix.rs | 2 +- vm/src/stdlib/pystruct.rs | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/vm/src/builtins/memory.rs b/vm/src/builtins/memory.rs index 4e3f23218d..eaf8539c77 100644 --- a/vm/src/builtins/memory.rs +++ b/vm/src/builtins/memory.rs @@ -11,7 +11,7 @@ use crate::{ protocol::{BufferInternal, BufferOptions, PyBuffer}, sliceable::{convert_slice, wrap_index, SequenceIndex}, slots::{AsBuffer, Comparable, Hashable, PyComparisonOp, SlotConstructor}, - stdlib::pystruct::_struct::FormatSpec, + stdlib::pystruct::FormatSpec, utils::Either, IdProtocol, IntoPyObject, PyClassImpl, PyComparisonValue, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromBorrowedObject, TryFromObject, TypeProtocol, VirtualMachine, diff --git a/vm/src/builtins/pystr.rs b/vm/src/builtins/pystr.rs index e29939296b..38be33ecde 100644 --- a/vm/src/builtins/pystr.rs +++ b/vm/src/builtins/pystr.rs @@ -477,7 +477,7 @@ impl PyStr { } #[pymethod(magic)] - pub(crate) fn repr(&self, vm: &VirtualMachine) -> PyResult { + pub fn repr(&self, vm: &VirtualMachine) -> PyResult { let in_len = self.byte_len(); let mut out_len = 0usize; // let mut max = 127; diff --git a/vm/src/lib.rs b/vm/src/lib.rs index 5b089dafea..e0ee8068e9 100644 --- a/vm/src/lib.rs +++ b/vm/src/lib.rs @@ -60,7 +60,7 @@ mod frozen; pub mod function; pub mod import; pub mod iterator; -mod protocol; +pub mod protocol; pub mod py_io; pub mod py_serde; mod pyobject; @@ -69,12 +69,12 @@ pub mod readline; pub mod scope; mod sequence; mod signal; -mod sliceable; +pub mod sliceable; pub mod slots; pub mod stdlib; pub mod types; pub mod utils; -mod version; +pub mod version; mod vm; // pub use self::Executor; diff --git a/vm/src/sliceable.rs b/vm/src/sliceable.rs index b366d9b4bf..0eb5fd1bea 100644 --- a/vm/src/sliceable.rs +++ b/vm/src/sliceable.rs @@ -404,7 +404,7 @@ pub(crate) fn wrap_index(p: isize, len: usize) -> Option { } // return pos is in range [0, len] inclusive -pub(crate) fn saturate_index(p: isize, len: usize) -> usize { +pub fn saturate_index(p: isize, len: usize) -> usize { let mut p = p; let len = len.to_isize().unwrap(); if p < 0 { diff --git a/vm/src/stdlib/io.rs b/vm/src/stdlib/io.rs index 548b652058..ba705f7044 100644 --- a/vm/src/stdlib/io.rs +++ b/vm/src/stdlib/io.rs @@ -35,7 +35,7 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyObjectRef { #[allow(unused)] #[derive(Copy, Clone)] #[repr(transparent)] -pub(crate) struct Fildes(pub i32); +pub struct Fildes(pub i32); impl TryFromObject for Fildes { fn try_from_object(vm: &VirtualMachine, obj: PyObjectRef) -> PyResult { diff --git a/vm/src/stdlib/posix.rs b/vm/src/stdlib/posix.rs index 90e3cc7f24..15961965a7 100644 --- a/vm/src/stdlib/posix.rs +++ b/vm/src/stdlib/posix.rs @@ -2,7 +2,7 @@ use crate::{PyObjectRef, PyResult, VirtualMachine}; use nix; use std::os::unix::io::RawFd; -pub(crate) fn raw_set_inheritable(fd: RawFd, inheritable: bool) -> nix::Result<()> { +pub fn raw_set_inheritable(fd: RawFd, inheritable: bool) -> nix::Result<()> { use nix::fcntl; let flags = fcntl::FdFlag::from_bits_truncate(fcntl::fcntl(fd, fcntl::FcntlArg::F_GETFD)?); let mut new_flags = flags; diff --git a/vm/src/stdlib/pystruct.rs b/vm/src/stdlib/pystruct.rs index a85a0f3f7d..085f73f92d 100644 --- a/vm/src/stdlib/pystruct.rs +++ b/vm/src/stdlib/pystruct.rs @@ -966,4 +966,4 @@ pub(crate) mod _struct { } } -pub(crate) use _struct::make_module; +pub(crate) use _struct::{make_module, FormatSpec}; From 6da45da5e7e62bb169100c19b3212a15c57971b3 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 30 Sep 2021 03:42:44 +0900 Subject: [PATCH 2/8] expose time::time as pub --- vm/src/stdlib/select.rs | 4 +-- vm/src/stdlib/time.rs | 64 +++++++++++++++++++++-------------------- 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/vm/src/stdlib/select.rs b/vm/src/stdlib/select.rs index a0411ca049..47ef9ffbfb 100644 --- a/vm/src/stdlib/select.rs +++ b/vm/src/stdlib/select.rs @@ -171,7 +171,7 @@ mod decl { return Err(vm.new_value_error("timeout must be positive".to_owned())); } } - let deadline = timeout.map(|s| time::get_time(vm).unwrap() + s); + let deadline = timeout.map(|s| time::time(vm).unwrap() + s); let seq2set = |list| -> PyResult<(Vec, FdSet)> { let v = vm.extract_elements::(list)?; @@ -210,7 +210,7 @@ mod decl { vm.check_signals()?; if let Some(ref mut timeout) = timeout { - *timeout = deadline.unwrap() - time::get_time(vm).unwrap(); + *timeout = deadline.unwrap() - time::time(vm).unwrap(); if *timeout < 0.0 { r.clear(); w.clear(); diff --git a/vm/src/stdlib/time.rs b/vm/src/stdlib/time.rs index 801c164b35..54a7409c45 100644 --- a/vm/src/stdlib/time.rs +++ b/vm/src/stdlib/time.rs @@ -2,7 +2,9 @@ // See also: // https://docs.python.org/3/library/time.html -use crate::{PyObjectRef, PyResult, VirtualMachine}; +use crate::{PyObjectRef, VirtualMachine}; + +pub use time::*; pub(crate) fn make_module(vm: &VirtualMachine) -> PyObjectRef { let module = time::make_module(vm); @@ -14,32 +16,6 @@ pub(crate) fn make_module(vm: &VirtualMachine) -> PyObjectRef { module } -#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] -pub(crate) fn get_time(vm: &VirtualMachine) -> PyResult { - Ok(duration_since_system_now(vm)?.as_secs_f64()) -} - -#[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] -pub(crate) fn get_time(_vm: &VirtualMachine) -> PyResult { - use wasm_bindgen::prelude::*; - #[wasm_bindgen] - extern "C" { - type Date; - #[wasm_bindgen(static_method_of = Date)] - fn now() -> f64; - } - // Date.now returns unix time in milliseconds, we want it in seconds - Ok(Date::now() / 1000.0) -} - -fn duration_since_system_now(vm: &VirtualMachine) -> PyResult { - use std::time::{SystemTime, UNIX_EPOCH}; - - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|e| vm.new_value_error(format!("Time error: {:?}", e))) -} - #[pymodule(name = "time")] mod time { use crate::{ @@ -70,6 +46,14 @@ mod time { #[allow(dead_code)] pub(super) const NS_TO_US: i64 = 1000; + fn duration_since_system_now(vm: &VirtualMachine) -> PyResult { + use std::time::{SystemTime, UNIX_EPOCH}; + + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|e| vm.new_value_error(format!("Time error: {:?}", e))) + } + #[cfg(not(unix))] #[pyfunction] fn sleep(dur: std::time::Duration) { @@ -79,19 +63,37 @@ mod time { #[cfg(not(target_os = "wasi"))] #[pyfunction] fn time_ns(vm: &VirtualMachine) -> PyResult { - Ok(super::duration_since_system_now(vm)?.as_nanos() as u64) + Ok(duration_since_system_now(vm)?.as_nanos() as u64) } #[pyfunction(name = "perf_counter")] // TODO: fix #[pyfunction] - fn time(vm: &VirtualMachine) -> PyResult { - super::get_time(vm) + pub fn time(vm: &VirtualMachine) -> PyResult { + _time(vm) + } + + #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] + fn _time(vm: &VirtualMachine) -> PyResult { + Ok(duration_since_system_now(vm)?.as_secs_f64()) + } + + #[cfg(all(target_arch = "wasm32", not(target_os = "wasi")))] + fn _time(_vm: &VirtualMachine) -> PyResult { + use wasm_bindgen::prelude::*; + #[wasm_bindgen] + extern "C" { + type Date; + #[wasm_bindgen(static_method_of = Date)] + fn now() -> f64; + } + // Date.now returns unix time in milliseconds, we want it in seconds + Ok(Date::now() / 1000.0) } #[pyfunction] fn monotonic(vm: &VirtualMachine) -> PyResult { // TODO: implement proper monotonic time! - Ok(super::duration_since_system_now(vm)?.as_secs_f64()) + Ok(duration_since_system_now(vm)?.as_secs_f64()) } fn pyobj_to_naive_date_time( From e888991da8b7e58db2a7342e8bfeb52ae72e4892 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 30 Sep 2021 03:44:26 +0900 Subject: [PATCH 3/8] expose PyTuple::_new for PyStructSequence --- derive/src/pystructseq.rs | 2 +- vm/src/builtins/pytype.rs | 2 +- vm/src/builtins/tuple.rs | 2 +- vm/src/pyobject.rs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/derive/src/pystructseq.rs b/derive/src/pystructseq.rs index 236f10efdd..0da5fa3831 100644 --- a/derive/src/pystructseq.rs +++ b/derive/src/pystructseq.rs @@ -36,7 +36,7 @@ pub(crate) fn impl_pystruct_sequence( self.#field_names, vm, )),*]; - ::rustpython_vm::builtins::tuple::PyTuple::_new(items.into_boxed_slice()) + ::rustpython_vm::builtins::tuple::PyTuple::new_unchecked(items.into_boxed_slice()) } } impl ::rustpython_vm::IntoPyObject for #ty { diff --git a/vm/src/builtins/pytype.rs b/vm/src/builtins/pytype.rs index 6d494ca5d3..71a27a327a 100644 --- a/vm/src/builtins/pytype.rs +++ b/vm/src/builtins/pytype.rs @@ -302,7 +302,7 @@ impl PyType { #[pyproperty(name = "__mro__")] fn get_mro(zelf: PyRef) -> PyTuple { let elements: Vec = zelf.iter_mro().map(|x| x.as_object().clone()).collect(); - PyTuple::_new(elements.into_boxed_slice()) + PyTuple::new_unchecked(elements.into_boxed_slice()) } #[pyproperty(magic)] diff --git a/vm/src/builtins/tuple.rs b/vm/src/builtins/tuple.rs index c10f30420b..65666cbf2e 100644 --- a/vm/src/builtins/tuple.rs +++ b/vm/src/builtins/tuple.rs @@ -115,7 +115,7 @@ impl PyTuple { /// Creating a new tuple with given boxed slice. /// NOTE: for usual case, you probably want to use PyTupleRef::with_elements. /// Calling this function implies trying micro optimization for non-zero-sized tuple. - pub(crate) fn _new(elements: Box<[PyObjectRef]>) -> Self { + pub fn new_unchecked(elements: Box<[PyObjectRef]>) -> Self { Self { elements } } diff --git a/vm/src/pyobject.rs b/vm/src/pyobject.rs index fe2ef73971..0eabfe0d76 100644 --- a/vm/src/pyobject.rs +++ b/vm/src/pyobject.rs @@ -106,7 +106,7 @@ impl PyContext { let false_value = create_object(PyInt::from(0), &types.bool_type); let empty_tuple = create_object( - PyTuple::_new(Vec::new().into_boxed_slice()), + PyTuple::new_unchecked(Vec::new().into_boxed_slice()), &types.tuple_type, ); let empty_frozenset = From b2ab17aaa706ef3ab9ba5bb4a9bcc6d108970fbb Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 30 Sep 2021 03:45:21 +0900 Subject: [PATCH 4/8] clean up to use only public api --- vm/src/stdlib/fcntl.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vm/src/stdlib/fcntl.rs b/vm/src/stdlib/fcntl.rs index 5a0b0a21b6..f0302dda83 100644 --- a/vm/src/stdlib/fcntl.rs +++ b/vm/src/stdlib/fcntl.rs @@ -3,7 +3,7 @@ pub(crate) use fcntl::make_module; #[pymodule] mod fcntl { use crate::{ - builtins::int, + builtins::PyIntRef, function::{ArgMemoryBuffer, ArgStrOrBytesLike, OptionalArg}, stdlib::{io, os}, utils::Either, @@ -25,7 +25,7 @@ mod fcntl { fn fcntl( io::Fildes(fd): io::Fildes, cmd: i32, - arg: OptionalArg>, + arg: OptionalArg>, vm: &VirtualMachine, ) -> PyResult { let int = match arg { From c62914daacb475b299b83e1e55892ebcf1407bc4 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 30 Sep 2021 05:31:52 +0900 Subject: [PATCH 5/8] expose PyFrozenSet --- vm/src/builtins/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/src/builtins/mod.rs b/vm/src/builtins/mod.rs index 29d96cd0e6..7931a31224 100644 --- a/vm/src/builtins/mod.rs +++ b/vm/src/builtins/mod.rs @@ -63,7 +63,7 @@ pub use pytype::{PyType, PyTypeRef}; pub(crate) mod range; pub use range::PyRange; pub(crate) mod set; -pub use set::PySet; +pub use set::{PyFrozenSet, PySet}; pub(crate) mod singletons; pub use singletons::{PyNone, PyNotImplemented}; pub(crate) mod slice; From d42c4eb21e32f324cdb0eeafff2dc1b2aed54a85 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Thu, 30 Sep 2021 03:01:45 +0900 Subject: [PATCH 6/8] move builtins module to vm::stdlib::builtlins for consistency --- vm/src/builtins/mod.rs | 3 --- vm/src/format.rs | 3 ++- vm/src/frame.rs | 5 ++--- .../make_module.rs => stdlib/builtins.rs} | 8 ++++---- vm/src/stdlib/mod.rs | 1 + vm/src/stdlib/sys.rs | 18 ++++++++++-------- vm/src/vm.rs | 7 +++---- 7 files changed, 22 insertions(+), 23 deletions(-) rename vm/src/{builtins/make_module.rs => stdlib/builtins.rs} (99%) diff --git a/vm/src/builtins/mod.rs b/vm/src/builtins/mod.rs index 7931a31224..9dc9f94ba9 100644 --- a/vm/src/builtins/mod.rs +++ b/vm/src/builtins/mod.rs @@ -84,7 +84,4 @@ pub use zip::PyZip; pub use float::try_to_bigint as try_f64_to_bigint; pub use int::try_to_float as try_bigint_to_f64; -mod make_module; -pub use make_module::{ascii, make_module, print}; - pub use crate::exceptions::types::*; diff --git a/vm/src/format.rs b/vm/src/format.rs index ddbda3ee99..30eb286393 100644 --- a/vm/src/format.rs +++ b/vm/src/format.rs @@ -1,8 +1,9 @@ use crate::{ - builtins::{self, PyBaseExceptionRef, PyStrRef}, + builtins::{PyBaseExceptionRef, PyStrRef}, common::float_ops, exceptions::IntoPyException, function::FuncArgs, + stdlib::builtins, ItemProtocol, PyObjectRef, PyResult, TypeProtocol, VirtualMachine, }; use itertools::{Itertools, PeekingNext}; diff --git a/vm/src/frame.rs b/vm/src/frame.rs index 9c7c28dab3..2d77e3eae6 100644 --- a/vm/src/frame.rs +++ b/vm/src/frame.rs @@ -1,8 +1,6 @@ use crate::common::{boxvec::BoxVec, lock::PyMutex}; use crate::{ - builtins::PyBaseExceptionRef, builtins::{ - self, asyncgenerator::PyAsyncGenWrappedValue, coroutine::PyCoroutine, function::{PyCell, PyCellRef, PyFunction}, @@ -10,7 +8,7 @@ use crate::{ list, pystr, set, traceback::PyTraceback, tuple::{PyTuple, PyTupleTyped}, - PyCode, PyDict, PyDictRef, PySlice, PyStr, PyStrRef, PyTypeRef, + PyBaseExceptionRef, PyCode, PyDict, PyDictRef, PySlice, PyStr, PyStrRef, PyTypeRef, }, bytecode, coroutine::Coro, @@ -20,6 +18,7 @@ use crate::{ protocol::PyIter, scope::Scope, slots::PyComparisonOp, + stdlib::builtins, IdProtocol, ItemProtocol, PyMethod, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, VirtualMachine, }; diff --git a/vm/src/builtins/make_module.rs b/vm/src/stdlib/builtins.rs similarity index 99% rename from vm/src/builtins/make_module.rs rename to vm/src/stdlib/builtins.rs index 964da58e6c..8edfea6189 100644 --- a/vm/src/builtins/make_module.rs +++ b/vm/src/stdlib/builtins.rs @@ -6,8 +6,8 @@ use crate::{PyObjectRef, VirtualMachine}; /// Built-in functions, exceptions, and other objects. /// /// Noteworthy: None is the `nil' object; Ellipsis represents `...' in slices. -#[pymodule(name = "builtins")] -mod decl { +#[pymodule] +mod builtins { use crate::builtins::{ enumerate::PyReverseSequenceIterator, function::{PyCellRef, PyFunctionRef}, @@ -891,12 +891,12 @@ mod decl { } } -pub use decl::{ascii, print}; +pub use builtins::{ascii, print}; pub fn make_module(vm: &VirtualMachine, module: PyObjectRef) { let ctx = &vm.ctx; - decl::extend_module(vm, &module); + builtins::extend_module(vm, &module); let debug_mode: bool = vm.state.settings.optimize == 0; extend_module!(vm, module, { diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index 8861c444ac..f2d73cc6b8 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -4,6 +4,7 @@ pub(crate) mod ast; mod atexit; mod binascii; mod bisect; +pub mod builtins; mod cmath; mod codecs; mod collections; diff --git a/vm/src/stdlib/sys.rs b/vm/src/stdlib/sys.rs index 11cd2d17db..fda309a85d 100644 --- a/vm/src/stdlib/sys.rs +++ b/vm/src/stdlib/sys.rs @@ -1,18 +1,20 @@ -use num_traits::ToPrimitive; -use std::{env, mem, path}; - -use crate::builtins::{PyStr, PyStrRef, PyTypeRef}; use crate::common::{ ascii, hash::{PyHash, PyUHash}, }; -use crate::frame::FrameRef; -use crate::function::{FuncArgs, OptionalArg, PosArgs}; -use crate::vm::{PySettings, VirtualMachine}; -use crate::{builtins, exceptions, py_io, version}; use crate::{ + builtins::{PyStr, PyStrRef, PyTypeRef}, + exceptions, + frame::FrameRef, + function::{FuncArgs, OptionalArg, PosArgs}, + py_io, + stdlib::builtins, + version, + vm::{PySettings, VirtualMachine}, ItemProtocol, PyClassImpl, PyContext, PyObjectRef, PyRefExact, PyResult, PyStructSequence, }; +use num_traits::ToPrimitive; +use std::{env, mem, path}; /* * The magic sys module. diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 45361bfb99..9aff2a72e0 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -8,13 +8,12 @@ use crate::compile::{self, CompileError, CompileErrorType, CompileOpts}; use crate::{ builtins::{ - self, code::{self, PyCode}, module, object, tuple::{PyTuple, PyTupleRef, PyTupleTyped}, - PyDictRef, PyInt, PyIntRef, PyList, PyModule, PyStr, PyStrRef, PyTypeRef, + PyBaseException, PyBaseExceptionRef, PyDictRef, PyInt, PyIntRef, PyList, PyModule, PyStr, + PyStrRef, PyTypeRef, }, - builtins::{PyBaseException, PyBaseExceptionRef}, bytecode, codecs::CodecsRegistry, common::{ascii, hash::HashSecret, lock::PyMutex, rc::PyRc}, @@ -325,7 +324,7 @@ impl VirtualMachine { panic!("Double Initialize Error"); } - builtins::make_module(self, self.builtins.clone()); + stdlib::builtins::make_module(self, self.builtins.clone()); stdlib::sys::make_module(self, self.sys_module.clone(), self.builtins.clone()); let mut inner_init = || -> PyResult<()> { From e881e3da0ab198ee2ba58d41b1700af44ae9684c Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Tue, 28 Sep 2021 16:56:55 +0900 Subject: [PATCH 7/8] move init_winsock to nt --- vm/src/stdlib/nt.rs | 8 ++++++++ vm/src/stdlib/select.rs | 4 +++- vm/src/stdlib/signal.rs | 2 +- vm/src/stdlib/socket.rs | 14 ++------------ 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index c55a42f590..3c08219d81 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -427,3 +427,11 @@ macro_rules! suppress_iph { ret }}; } + +pub fn init_winsock() { + static WSA_INIT: parking_lot::Once = parking_lot::Once::new(); + WSA_INIT.call_once(|| unsafe { + let mut wsa_data = std::mem::MaybeUninit::uninit(); + let _ = winapi::um::winsock2::WSAStartup(0x0101, wsa_data.as_mut_ptr()); + }) +} diff --git a/vm/src/stdlib/select.rs b/vm/src/stdlib/select.rs index 47ef9ffbfb..33a2a15e34 100644 --- a/vm/src/stdlib/select.rs +++ b/vm/src/stdlib/select.rs @@ -2,7 +2,9 @@ use crate::{PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, Virtual use std::{io, mem}; pub(crate) fn make_module(vm: &VirtualMachine) -> PyObjectRef { - super::socket::init_winsock(); + #[cfg(windows)] + super::nt::init_winsock(); + #[cfg(unix)] { use crate::PyClassImpl; diff --git a/vm/src/stdlib/signal.rs b/vm/src/stdlib/signal.rs index 0eff5d1584..c696012d41 100644 --- a/vm/src/stdlib/signal.rs +++ b/vm/src/stdlib/signal.rs @@ -192,7 +192,7 @@ pub(crate) mod _signal { #[cfg(windows)] let is_socket = if fd != INVALID_WAKEUP { - crate::stdlib::socket::init_winsock(); + crate::stdlib::nt::init_winsock(); let mut res = 0i32; let mut res_size = std::mem::size_of::() as i32; let res = unsafe { diff --git a/vm/src/stdlib/socket.rs b/vm/src/stdlib/socket.rs index 5dd94b1368..63b8778960 100644 --- a/vm/src/stdlib/socket.rs +++ b/vm/src/stdlib/socket.rs @@ -1818,7 +1818,8 @@ rustpython_common::static_cell! { } pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { - init_winsock(); + #[cfg(windows)] + super::nt::init_winsock(); let ctx = &vm.ctx; let socket_timeout = TIMEOUT_ERROR @@ -1978,14 +1979,3 @@ fn extend_module_platform_specific(vm: &VirtualMachine, module: &PyObjectRef) { "sethostname" => named_function!(ctx, _socket, sethostname), }); } - -pub fn init_winsock() { - #[cfg(windows)] - { - static WSA_INIT: parking_lot::Once = parking_lot::Once::new(); - WSA_INIT.call_once(|| unsafe { - let mut wsa_data = std::mem::MaybeUninit::uninit(); - let _ = winapi::um::winsock2::WSAStartup(0x0101, wsa_data.as_mut_ptr()); - }) - } -} From fb25a15f2c9ed41e638002920eaad32ae610f6df Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Wed, 22 Sep 2021 03:13:56 +0900 Subject: [PATCH 8/8] split rustpython-stdlib crate --- Cargo.lock | 141 +++++++++++------- Cargo.toml | 13 +- src/lib.rs | 7 + stdlib/Cargo.toml | 110 ++++++++++++++ stdlib/build.rs | 28 ++++ {vm/src/stdlib => stdlib/src}/array.rs | 5 +- {vm/src/stdlib => stdlib/src}/binascii.rs | 4 +- {vm/src/stdlib => stdlib/src}/bisect.rs | 2 +- {vm/src/stdlib => stdlib/src}/cmath.rs | 2 +- {vm/src/stdlib => stdlib/src}/csv.rs | 4 +- {vm/src/stdlib => stdlib/src}/dis.rs | 2 +- {vm/src/stdlib => stdlib/src}/faulthandler.rs | 2 +- {vm/src/stdlib => stdlib/src}/fcntl.rs | 2 +- {vm/src/stdlib => stdlib/src}/hashlib.rs | 7 +- {vm/src/stdlib => stdlib/src}/json.rs | 2 +- .../stdlib => stdlib/src}/json/machinery.rs | 0 {vm/src/stdlib => stdlib/src}/keyword.rs | 2 +- stdlib/src/lib.rs | 134 +++++++++++++++++ {vm/src/stdlib => stdlib/src}/math.rs | 2 +- .../stdlib => stdlib/src}/multiprocessing.rs | 2 +- {vm/src/stdlib => stdlib/src}/platform.rs | 2 +- .../stdlib => stdlib/src}/posixsubprocess.rs | 4 +- {vm/src/stdlib => stdlib/src}/pyexpat.rs | 5 +- {vm/src/stdlib => stdlib/src}/random.rs | 2 +- {vm/src/stdlib => stdlib/src}/re.rs | 0 {vm/src/stdlib => stdlib/src}/resource.rs | 2 +- {vm/src/stdlib => stdlib/src}/scproxy.rs | 2 +- {vm/src/stdlib => stdlib/src}/select.rs | 10 +- {vm/src/stdlib => stdlib/src}/socket.rs | 30 ++-- {vm/src/stdlib => stdlib/src}/ssl.rs | 9 +- {vm/src/stdlib => stdlib/src}/syslog.rs | 2 +- {vm/src/stdlib => stdlib/src}/termios.rs | 2 +- {vm/src/stdlib => stdlib/src}/unicodedata.rs | 6 +- {vm/src/stdlib => stdlib/src}/zlib.rs | 2 +- vm/Cargo.toml | 46 +----- vm/build.rs | 27 ---- vm/src/builtins/set.rs | 2 +- vm/src/exceptions.rs | 2 +- vm/src/import.rs | 2 +- vm/src/protocol/buffer.rs | 2 +- vm/src/protocol/mod.rs | 2 +- vm/src/stdlib/imp.rs | 4 +- vm/src/stdlib/mod.rs | 91 +---------- vm/src/stdlib/nt.rs | 7 +- vm/src/stdlib/sys.rs | 2 +- vm/src/vm.rs | 8 +- 46 files changed, 462 insertions(+), 282 deletions(-) create mode 100644 stdlib/Cargo.toml create mode 100644 stdlib/build.rs rename {vm/src/stdlib => stdlib/src}/array.rs (99%) rename {vm/src/stdlib => stdlib/src}/binascii.rs (98%) rename {vm/src/stdlib => stdlib/src}/bisect.rs (99%) rename {vm/src/stdlib => stdlib/src}/cmath.rs (99%) rename {vm/src/stdlib => stdlib/src}/csv.rs (99%) rename {vm/src/stdlib => stdlib/src}/dis.rs (98%) rename {vm/src/stdlib => stdlib/src}/faulthandler.rs (94%) rename {vm/src/stdlib => stdlib/src}/fcntl.rs (99%) rename {vm/src/stdlib => stdlib/src}/hashlib.rs (98%) rename {vm/src/stdlib => stdlib/src}/json.rs (99%) rename {vm/src/stdlib => stdlib/src}/json/machinery.rs (100%) rename {vm/src/stdlib => stdlib/src}/keyword.rs (90%) create mode 100644 stdlib/src/lib.rs rename {vm/src/stdlib => stdlib/src}/math.rs (99%) rename {vm/src/stdlib => stdlib/src}/multiprocessing.rs (93%) rename {vm/src/stdlib => stdlib/src}/platform.rs (94%) rename {vm/src/stdlib => stdlib/src}/posixsubprocess.rs (98%) rename {vm/src/stdlib => stdlib/src}/pyexpat.rs (98%) rename {vm/src/stdlib => stdlib/src}/random.rs (99%) rename {vm/src/stdlib => stdlib/src}/re.rs (100%) rename {vm/src/stdlib => stdlib/src}/resource.rs (99%) rename {vm/src/stdlib => stdlib/src}/scproxy.rs (99%) rename {vm/src/stdlib => stdlib/src}/select.rs (98%) rename {vm/src/stdlib => stdlib/src}/socket.rs (98%) rename {vm/src/stdlib => stdlib/src}/ssl.rs (99%) rename {vm/src/stdlib => stdlib/src}/syslog.rs (99%) rename {vm/src/stdlib => stdlib/src}/termios.rs (99%) rename {vm/src/stdlib => stdlib/src}/unicodedata.rs (96%) rename {vm/src/stdlib => stdlib/src}/zlib.rs (99%) diff --git a/Cargo.lock b/Cargo.lock index 520b6c14b0..ffb2e02d57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,9 +146,9 @@ checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" [[package]] name = "blake2" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a5720225ef5daecf08657f23791354e1685a8c91a4c60c7f3d3b2892f978f4" +checksum = "0a4e37d16930f5459780f5621038b6382b9bb37c19016f39fb6b5808d831f174" dependencies = [ "crypto-mac", "digest", @@ -320,9 +320,9 @@ checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" [[package]] name = "cpufeatures" -version = "0.1.5" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef" +checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" dependencies = [ "libc", ] @@ -1035,9 +1035,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.101" +version = "0.2.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21" +checksum = "a2a5ac8f984bfcf3a823267e5fde638acc3325f6496633a5da6bb6eb2171e103" [[package]] name = "libffi" @@ -1267,9 +1267,9 @@ dependencies = [ [[package]] name = "num_enum" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2c8fd66061a707503d515639b8af10fd3807a5b5ee6959f7ff1bd303634bd5" +checksum = "3f9bd055fb730c4f8f4f57d45d35cd6b3f0980535b056dc7ff119cee6a66ed6f" dependencies = [ "derivative", "num_enum_derive", @@ -1277,9 +1277,9 @@ dependencies = [ [[package]] name = "num_enum_derive" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "474fd1d096da3ad17084694eebed40ba09c4a36c5255cd772bd8b98859cc562e" +checksum = "486ea01961c4a818096de679a8b740b26d9033146ac5291b1c98557658f8cdd9" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -1327,9 +1327,9 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" [[package]] name = "openssl-src" -version = "111.15.0+1.1.1k" +version = "111.16.0+1.1.1l" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a5f6ae2ac04393b217ea9f700cd04fa9bf3d93fae2872069f3d15d908af70a" +checksum = "7ab2173f69416cf3ec12debb5823d244127d23a9b127d5a5189aa97c5fa2859f" dependencies = [ "cc", ] @@ -1750,6 +1750,7 @@ dependencies = [ "rustpython-compiler", "rustpython-parser", "rustpython-pylib", + "rustpython-stdlib", "rustpython-vm", "rustyline", ] @@ -1881,32 +1882,88 @@ dependencies = [ ] [[package]] -name = "rustpython-vm" +name = "rustpython-stdlib" version = "0.1.2" dependencies = [ "adler32", "ahash", "ascii", - "atty", "base64", - "bitflags", "blake2", - "bstr", - "caseless", "cfg-if 1.0.0", - "chrono", "crc", "crc32fast", "crossbeam-utils", "csv-core", "digest", "dns-lookup", + "flate2", + "foreign-types-shared", + "gethostname", + "hex", + "itertools", + "lexical-core", + "libc", + "libz-sys", + "md-5", + "memchr", + "mt19937", + "nix", + "num-bigint", + "num-complex", + "num-integer", + "num-traits", + "num_enum", + "openssl", + "openssl-probe", + "openssl-sys", + "paste", + "puruspe", + "rand", + "rand_core", + "rustpython-common", + "rustpython-derive", + "rustpython-parser", + "rustpython-vm", + "schannel", + "sha-1", + "sha2", + "sha3", + "socket2", + "system-configuration", + "termios", + "unic-char-property", + "unic-normal", + "unic-ucd-age", + "unic-ucd-bidi", + "unic-ucd-category", + "unic-ucd-ident", + "unicode-casing", + "unicode_names2", + "widestring", + "winapi", + "xml-rs", +] + +[[package]] +name = "rustpython-vm" +version = "0.1.2" +dependencies = [ + "adler32", + "ahash", + "ascii", + "atty", + "bitflags", + "bstr", + "caseless", + "cfg-if 1.0.0", + "chrono", + "crc32fast", + "crossbeam-utils", "exitcode", "flame", "flamer", "flate2", - "foreign-types-shared", - "gethostname", "getrandom", "half", "hex", @@ -1916,11 +1973,8 @@ dependencies = [ "itertools", "lexical-core", "libc", - "libz-sys", "log", - "md-5", "memchr", - "mt19937", "nix", "num-bigint", "num-complex", @@ -1930,14 +1984,9 @@ dependencies = [ "num_cpus", "num_enum", "once_cell", - "openssl", - "openssl-probe", - "openssl-sys", "parking_lot", "paste", - "puruspe", "rand", - "rand_core", "result-like", "rustc_version", "rustpython-ast", @@ -1950,25 +1999,15 @@ dependencies = [ "rustpython-parser", "rustpython-pylib", "rustyline", - "schannel", "serde", - "sha-1", - "sha2", - "sha3", - "socket2", "sre-engine", "static_assertions", "strum", "strum_macros", - "system-configuration", - "termios", "thiserror", "thread_local", "timsort", "uname", - "unic-char-property", - "unic-normal", - "unic-ucd-age", "unic-ucd-bidi", "unic-ucd-category", "unic-ucd-ident", @@ -1979,7 +2018,6 @@ dependencies = [ "widestring", "winapi", "winreg", - "xml-rs", ] [[package]] @@ -2133,9 +2171,9 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a0c8611594e2ab4ebbf06ec7cbbf0a99450b8570e96cbf5188b5d5f6ef18d81" +checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" dependencies = [ "block-buffer", "cfg-if 1.0.0", @@ -2146,9 +2184,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.9.5" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" +checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa" dependencies = [ "block-buffer", "cfg-if 1.0.0", @@ -2269,9 +2307,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.74" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1873d832550d4588c3dbc20f01361ab00bfe741048f71e3fecf145a7cc18b29c" +checksum = "b7f58f7e8eaa0009c5fec437aabf511bd9933e4b2d7407bd05273c01a8906ea7" dependencies = [ "proc-macro2", "quote", @@ -2414,11 +2452,12 @@ dependencies = [ [[package]] name = "time" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" dependencies = [ "libc", + "wasi", "winapi", ] @@ -2473,9 +2512,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.13.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" +checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" [[package]] name = "uname" @@ -2690,9 +2729,9 @@ dependencies = [ [[package]] name = "wasi" -version = "0.10.2+wasi-snapshot-preview1" +version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" [[package]] name = "wasm-bindgen" diff --git a/Cargo.toml b/Cargo.toml index d9e875cfb1..68e0e7968d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,19 +12,19 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"] resolver = "2" members = [ ".", "ast", "bytecode", "common", "compiler", "compiler/porcelain", - "derive", "jit", "parser", "vm", "vm/pylib-crate", "wasm/lib", + "derive", "jit", "parser", "vm", "vm/pylib-crate", "stdlib", "wasm/lib", ] [features] -default = ["threading", "pylib", "zlib"] +default = ["threading", "pylib", "zlib", "stdlib"] +stdlib = ["rustpython-stdlib"] flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"] freeze-stdlib = ["rustpython-vm/freeze-stdlib"] jit = ["rustpython-vm/jit"] threading = ["rustpython-vm/threading"] -zlib = ["rustpython-vm/zlib"] - -ssl = ["rustpython-vm/ssl"] -ssl-vendor = ["rustpython-vm/ssl-vendor"] +zlib = ["rustpython-stdlib/zlib"] +ssl = ["rustpython-stdlib/ssl"] +ssl-vendor = ["rustpython-stdlib/ssl-vendor"] [dependencies] log = "0.4" @@ -33,6 +33,7 @@ clap = "2.33" rustpython-compiler = { path = "compiler/porcelain", version = "0.1.1" } rustpython-parser = { path = "parser", version = "0.1.1" } rustpython-vm = { path = "vm", version = "0.1.1", default-features = false, features = ["compile-parse"] } +rustpython-stdlib = {path = "stdlib", optional = true, default-features = false, features = ["compile-parse"]} pylib = { package = "rustpython-pylib", path = "vm/pylib-crate", version = "0.1.0", optional = true } dirs = { package = "dirs-next", version = "2.0.0" } num-traits = "0.2.8" diff --git a/src/lib.rs b/src/lib.rs index d820c0013a..ea9a9b74b9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -93,6 +93,13 @@ where }; let interp = Interpreter::new_with_init(settings, |vm| { + #[cfg(feature = "stdlib")] + { + let stdlib = rustpython_stdlib::get_module_inits(); + for (name, init) in stdlib.into_iter() { + vm.add_native_module(name, init); + } + } init(vm); init_param }); diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml new file mode 100644 index 0000000000..a4c09627f1 --- /dev/null +++ b/stdlib/Cargo.toml @@ -0,0 +1,110 @@ +[package] +name = "rustpython-stdlib" +version = "0.1.2" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rustpython-parser = { path = "../parser", optional = true } +rustpython-derive = { path = "../derive" } +rustpython-vm = { path = "../vm" } +rustpython-common = { path = "../common" } + +# random +rand = "0.8" +rand_core = "0.6" +mt19937 = "2.0" + +# Crypto: +digest = "0.9.0" +md-5 = "0.9.1" +sha-1 = "0.9.4" +sha2 = "0.9.3" +sha3 = "0.9.1" +blake2 = "0.9.1" + +## unicode stuff +unicode_names2 = "0.4" +# TODO: use unic for this; needed for title case: +# https://github.com/RustPython/RustPython/pull/832#discussion_r275428939 +unicode-casing = "0.1" +# update version all at the same time +unic-char-property = "0.9" +unic-normal = "0.9" +unic-ucd-bidi = "0.9" +unic-ucd-category = "0.9" +unic-ucd-age = "0.9" +unic-ucd-ident = "0.9" + +# compression +adler32 = "1.0.3" +crc32fast = "1.2.0" +flate2 = "1.0.20" + +num-complex = "0.4.0" +num-bigint = "0.4.2" +num-integer = "0.1.44" + +crossbeam-utils = "0.8.5" +itertools = "0.10.1" +lexical-core = "0.7" +num-traits = "0.2.14" +crc = "^1.0.0" +memchr = "2" +base64 = "0.13" +csv-core = "0.1" +hex = "0.4.0" +puruspe = "0.1" +nix = "0.22.1" +xml-rs = "0.8" +libc = "0.2.102" +cfg-if = "1.0.0" +ahash = "0.7.4" +libz-sys = { version = "1.0", optional = true } +num_enum = "0.5.4" +ascii = "1.0.0" + +[target.'cfg(all(unix, not(target_os = "redox")))'.dependencies] +termios = "0.3" + +[features] +default = ["compile-parse", "pylib"] +pylib = ["rustpython-vm/pylib"] +compile-parse = ["rustpython-vm/compile-parse", "rustpython-parser"] +threading = ["rustpython-vm/threading"] +parser = ["rustpython-vm/parser"] +ast = ["rustpython-vm/ast"] +compiler = ["rustpython-vm/compiler"] +# compiler = ["rustpython-compiler", "rustpython-compiler-core", "ast"] +# parser = ["rustpython-parser", "ast"] + +# TODO: use resolver = "2" instead of features +zlib = ["libz-sys", "flate2/zlib"] +ssl = ["openssl", "openssl-sys", "foreign-types-shared"] +ssl-vendor = ["ssl", "openssl/vendored", "openssl-probe"] + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +gethostname = "0.2.0" +socket2 = { version = "0.4.0", features = ["all"] } +dns-lookup = "1.0" +openssl = { version = "0.10.32", optional = true } +openssl-sys = { version = "0.9", optional = true } +openssl-probe = { version = "0.1", optional = true } +foreign-types-shared = { version = "0.1", optional = true } + +[target.'cfg(windows)'.dependencies] +schannel = "0.1" +widestring = "0.4" +paste = "1.0.5" + +[target.'cfg(windows)'.dependencies.winapi] +version = "0.3" +features = [ + "winsock2", "handleapi", "ws2def", "std", "winbase", "wincrypt", "fileapi", "processenv", + "namedpipeapi", "winnt", "processthreadsapi", "errhandlingapi", "winuser", "synchapi", "wincon", + "impl-default", "vcruntime", "ifdef", "netioapi" +] + +[target.'cfg(target_os = "macos")'.dependencies] +system-configuration = "0.4" diff --git a/stdlib/build.rs b/stdlib/build.rs new file mode 100644 index 0000000000..135f71152a --- /dev/null +++ b/stdlib/build.rs @@ -0,0 +1,28 @@ +fn main() { + #[allow(clippy::unusual_byte_groupings)] + if let Ok(v) = std::env::var("DEP_OPENSSL_VERSION_NUMBER") { + println!("cargo:rustc-env=OPENSSL_API_VERSION={}", v); + // cfg setup from openssl crate's build script + let version = u64::from_str_radix(&v, 16).unwrap(); + if version >= 0x1_00_01_00_0 { + println!("cargo:rustc-cfg=ossl101"); + } + if version >= 0x1_00_02_00_0 { + println!("cargo:rustc-cfg=ossl102"); + } + if version >= 0x1_01_00_00_0 { + println!("cargo:rustc-cfg=ossl110"); + } + if version >= 0x1_01_00_07_0 { + println!("cargo:rustc-cfg=ossl110g"); + } + if version >= 0x1_01_01_00_0 { + println!("cargo:rustc-cfg=ossl111"); + } + } + if let Ok(v) = std::env::var("DEP_OPENSSL_CONF") { + for conf in v.split(',') { + println!("cargo:rustc-cfg=osslconf=\"{}\"", conf); + } + } +} diff --git a/vm/src/stdlib/array.rs b/stdlib/src/array.rs similarity index 99% rename from vm/src/stdlib/array.rs rename to stdlib/src/array.rs index 58175cb3f5..262ea06f59 100644 --- a/vm/src/stdlib/array.rs +++ b/stdlib/src/array.rs @@ -10,11 +10,12 @@ mod array { }, str::wchar_t, }; - use crate::{ + use crate::vm::{ builtins::{ IntoPyFloat, PyByteArray, PyBytes, PyBytesRef, PyIntRef, PyList, PyListRef, PySliceRef, PyStr, PyStrRef, PyTypeRef, }, + class_or_notimplemented, function::{ArgBytesLike, ArgIterable, OptionalArg}, protocol::{BufferInternal, BufferOptions, PyBuffer, ResizeGuard}, sliceable::{saturate_index, PySliceableSequence, PySliceableSequenceMut, SequenceIndex}, @@ -36,7 +37,7 @@ mod array { macro_rules! def_array_enum { ($(($n:ident, $t:ty, $c:literal, $scode:literal)),*$(,)?) => { #[derive(Debug, Clone)] - pub(crate) enum ArrayContentType { + pub enum ArrayContentType { $($n(Vec<$t>),)* } diff --git a/vm/src/stdlib/binascii.rs b/stdlib/src/binascii.rs similarity index 98% rename from vm/src/stdlib/binascii.rs rename to stdlib/src/binascii.rs index 035f16ccd3..7b0e4f94fe 100644 --- a/vm/src/stdlib/binascii.rs +++ b/stdlib/src/binascii.rs @@ -2,10 +2,10 @@ pub(crate) use decl::make_module; #[pymodule(name = "binascii")] mod decl { - use crate::{ + use crate::vm::{ builtins::{PyByteArray, PyBytes, PyStr, PyTypeRef}, function::{ArgBytesLike, OptionalArg}, - PyObjectRef, PyRef, PyResult, TryFromObject, TypeProtocol, VirtualMachine, + match_class, PyObjectRef, PyRef, PyResult, TryFromObject, TypeProtocol, VirtualMachine, }; use crc::{crc32, Hasher32}; use itertools::Itertools; diff --git a/vm/src/stdlib/bisect.rs b/stdlib/src/bisect.rs similarity index 99% rename from vm/src/stdlib/bisect.rs rename to stdlib/src/bisect.rs index dd03199186..af51088436 100644 --- a/vm/src/stdlib/bisect.rs +++ b/stdlib/src/bisect.rs @@ -2,7 +2,7 @@ pub(crate) use _bisect::make_module; #[pymodule] mod _bisect { - use crate::{ + use crate::vm::{ function::OptionalArg, slots::PyComparisonOp::Lt, ItemProtocol, PyObjectRef, PyResult, VirtualMachine, }; diff --git a/vm/src/stdlib/cmath.rs b/stdlib/src/cmath.rs similarity index 99% rename from vm/src/stdlib/cmath.rs rename to stdlib/src/cmath.rs index e337510e20..e3f4d1ec4c 100644 --- a/vm/src/stdlib/cmath.rs +++ b/stdlib/src/cmath.rs @@ -5,7 +5,7 @@ pub(crate) use cmath::make_module; /// This module provides access to mathematical functions for complex numbers. #[pymodule] mod cmath { - use crate::{ + use crate::vm::{ builtins::{IntoPyComplex, IntoPyFloat}, function::OptionalArg, PyResult, VirtualMachine, diff --git a/vm/src/stdlib/csv.rs b/stdlib/src/csv.rs similarity index 99% rename from vm/src/stdlib/csv.rs rename to stdlib/src/csv.rs index fab0a02ef1..462df4c794 100644 --- a/vm/src/stdlib/csv.rs +++ b/stdlib/src/csv.rs @@ -1,8 +1,10 @@ use crate::common::lock::PyMutex; -use crate::{ +use crate::vm::{ builtins::{PyStr, PyStrRef}, function::{ArgIterable, ArgumentError, FromArgs, FuncArgs}, + match_class, named_function, protocol::PyIter, + py_module, slots::{IteratorIterable, SlotIterator}, types::create_simple_type, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, TypeProtocol, diff --git a/vm/src/stdlib/dis.rs b/stdlib/src/dis.rs similarity index 98% rename from vm/src/stdlib/dis.rs rename to stdlib/src/dis.rs index 56c2f6c37c..8ca7eabab8 100644 --- a/vm/src/stdlib/dis.rs +++ b/stdlib/src/dis.rs @@ -2,7 +2,7 @@ pub(crate) use decl::make_module; #[pymodule(name = "dis")] mod decl { - use crate::{ + use crate::vm::{ builtins::{PyCode, PyDictRef, PyStrRef}, bytecode::CodeFlags, compile, ItemProtocol, PyObjectRef, PyRef, PyResult, TryFromObject, VirtualMachine, diff --git a/vm/src/stdlib/faulthandler.rs b/stdlib/src/faulthandler.rs similarity index 94% rename from vm/src/stdlib/faulthandler.rs rename to stdlib/src/faulthandler.rs index 8a7019ddf7..17f0498c93 100644 --- a/vm/src/stdlib/faulthandler.rs +++ b/stdlib/src/faulthandler.rs @@ -2,7 +2,7 @@ pub(crate) use decl::make_module; #[pymodule(name = "faulthandler")] mod decl { - use crate::{frame::FrameRef, function::OptionalArg, VirtualMachine}; + use crate::vm::{frame::FrameRef, function::OptionalArg, VirtualMachine}; fn dump_frame(frame: &FrameRef) { eprintln!( diff --git a/vm/src/stdlib/fcntl.rs b/stdlib/src/fcntl.rs similarity index 99% rename from vm/src/stdlib/fcntl.rs rename to stdlib/src/fcntl.rs index f0302dda83..0fa64b0e8e 100644 --- a/vm/src/stdlib/fcntl.rs +++ b/stdlib/src/fcntl.rs @@ -2,7 +2,7 @@ pub(crate) use fcntl::make_module; #[pymodule] mod fcntl { - use crate::{ + use crate::vm::{ builtins::PyIntRef, function::{ArgMemoryBuffer, ArgStrOrBytesLike, OptionalArg}, stdlib::{io, os}, diff --git a/vm/src/stdlib/hashlib.rs b/stdlib/src/hashlib.rs similarity index 98% rename from vm/src/stdlib/hashlib.rs rename to stdlib/src/hashlib.rs index 1fc3c96801..9ae9cab30a 100644 --- a/vm/src/stdlib/hashlib.rs +++ b/stdlib/src/hashlib.rs @@ -3,7 +3,7 @@ pub(crate) use hashlib::make_module; #[pymodule] mod hashlib { use crate::common::lock::{PyRwLock, PyRwLockReadGuard, PyRwLockWriteGuard}; - use crate::{ + use crate::vm::{ builtins::{PyBytes, PyBytesRef, PyStrRef, PyTypeRef}, function::{FuncArgs, OptionalArg}, PyResult, PyValue, VirtualMachine, @@ -14,7 +14,6 @@ mod hashlib { use sha1::Sha1; use sha2::{Sha224, Sha256, Sha384, Sha512}; use sha3::{Sha3_224, Sha3_256, Sha3_384, Sha3_512}; // TODO: , Shake128, Shake256; - use std::fmt; #[pyattr] #[pyclass(module = "hashlib", name = "hasher")] @@ -24,8 +23,8 @@ mod hashlib { buffer: PyRwLock, } - impl fmt::Debug for PyHasher { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + impl std::fmt::Debug for PyHasher { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { write!(f, "hasher {}", self.name) } } diff --git a/vm/src/stdlib/json.rs b/stdlib/src/json.rs similarity index 99% rename from vm/src/stdlib/json.rs rename to stdlib/src/json.rs index daaf7e8b9b..a699d2971f 100644 --- a/vm/src/stdlib/json.rs +++ b/stdlib/src/json.rs @@ -4,7 +4,7 @@ mod machinery; #[pymodule] mod _json { use super::machinery; - use crate::{ + use crate::vm::{ builtins::PyBaseExceptionRef, builtins::{PyStrRef, PyTypeRef}, function::{FuncArgs, OptionalArg}, diff --git a/vm/src/stdlib/json/machinery.rs b/stdlib/src/json/machinery.rs similarity index 100% rename from vm/src/stdlib/json/machinery.rs rename to stdlib/src/json/machinery.rs diff --git a/vm/src/stdlib/keyword.rs b/stdlib/src/keyword.rs similarity index 90% rename from vm/src/stdlib/keyword.rs rename to stdlib/src/keyword.rs index 7012a91f04..5f4b825f4b 100644 --- a/vm/src/stdlib/keyword.rs +++ b/stdlib/src/keyword.rs @@ -3,7 +3,7 @@ pub(crate) use decl::make_module; #[pymodule(name = "keyword")] mod decl { - use crate::{builtins::PyStr, PyObjectRef, VirtualMachine}; + use crate::vm::{builtins::PyStr, PyObjectRef, VirtualMachine}; use itertools::Itertools; use rustpython_parser::lexer; diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs new file mode 100644 index 0000000000..c29897b54d --- /dev/null +++ b/stdlib/src/lib.rs @@ -0,0 +1,134 @@ +// to allow `mod foo {}` in foo.rs; clippy thinks this is a mistake/misunderstanding of +// how `mod` works, but we want this sometimes for pymodule declarations +#![allow(clippy::module_inception)] + +#[macro_use] +extern crate rustpython_derive; + +pub mod array; +mod binascii; +mod bisect; +mod cmath; +mod csv; +mod dis; +mod hashlib; +mod json; +#[cfg(feature = "rustpython-parser")] +mod keyword; +mod math; +mod platform; +mod pyexpat; +mod random; +// TODO: maybe make this an extension module, if we ever get those +// mod re; +#[cfg(not(target_arch = "wasm32"))] +pub mod socket; +#[cfg(unix)] +mod syslog; +mod unicodedata; +mod zlib; + +#[cfg(not(target_arch = "wasm32"))] +mod faulthandler; +#[cfg(any(unix, target_os = "wasi"))] +mod fcntl; +#[cfg(not(target_arch = "wasm32"))] +mod multiprocessing; +#[cfg(unix)] +mod posixsubprocess; +// libc is missing constants on redox +#[cfg(all(unix, not(target_os = "redox")))] +mod resource; +#[cfg(target_os = "macos")] +mod scproxy; +#[cfg(not(target_arch = "wasm32"))] +mod select; +#[cfg(all(not(target_arch = "wasm32"), feature = "ssl"))] +mod ssl; +#[cfg(all(unix, not(target_os = "redox")))] +mod termios; + +use rustpython_common as common; +use rustpython_vm as vm; + +use crate::vm::{ + builtins, + stdlib::{StdlibInitFunc, StdlibMap}, +}; +use std::borrow::Cow; + +pub fn get_module_inits() -> StdlibMap { + macro_rules! modules { + { + $( + #[cfg($cfg:meta)] + { $( $key:expr => $val:expr),* $(,)? } + )* + } => {{ + let iter = std::array::IntoIter::new([ + $( + $(#[cfg($cfg)] (Cow::<'static, str>::from($key), Box::new($val) as StdlibInitFunc),)* + )* + ]); + iter.collect() + }}; + } + modules! { + #[cfg(all())] + { + "array" => array::make_module, + "binascii" => binascii::make_module, + "_bisect" => bisect::make_module, + "cmath" => cmath::make_module, + "_csv" => csv::make_module, + "dis" => dis::make_module, + "hashlib" => hashlib::make_module, + "_json" => json::make_module, + "math" => math::make_module, + "pyexpat" => pyexpat::make_module, + "_platform" => platform::make_module, + "_random" => random::make_module, + "unicodedata" => unicodedata::make_module, + "zlib" => zlib::make_module, + // crate::vm::sysmodule::sysconfigdata_name() => sysconfigdata::make_module, + } + // parser related modules: + #[cfg(feature = "rustpython-ast")] + { + "_ast" => ast::make_module, + } + #[cfg(feature = "rustpython-parser")] + { + "keyword" => keyword::make_module, + } + #[cfg(any(unix, target_os = "wasi"))] + { + "fcntl" => fcntl::make_module, + } + #[cfg(not(target_arch = "wasm32"))] + { + "_multiprocessing" => multiprocessing::make_module, + "select" => select::make_module, + "_socket" => socket::make_module, + "faulthandler" => faulthandler::make_module, + } + #[cfg(feature = "ssl")] + { + "_ssl" => ssl::make_module, + } + #[cfg(all(unix, not(target_os = "redox")))] + { + "termios" => termios::make_module, + "resource" => resource::make_module, + } + #[cfg(unix)] + { + "_posixsubprocess" => posixsubprocess::make_module, + "syslog" => syslog::make_module, + } + #[cfg(target_os = "macos")] + { + "_scproxy" => scproxy::make_module, + } + } +} diff --git a/vm/src/stdlib/math.rs b/stdlib/src/math.rs similarity index 99% rename from vm/src/stdlib/math.rs rename to stdlib/src/math.rs index 0b753b31bb..1f2bfa3508 100644 --- a/vm/src/stdlib/math.rs +++ b/stdlib/src/math.rs @@ -2,7 +2,7 @@ pub(crate) use math::make_module; #[pymodule] mod math { - use crate::{ + use crate::vm::{ builtins::{ try_bigint_to_f64, try_f64_to_bigint, IntoPyFloat, PyFloatRef, PyInt, PyIntRef, }, diff --git a/vm/src/stdlib/multiprocessing.rs b/stdlib/src/multiprocessing.rs similarity index 93% rename from vm/src/stdlib/multiprocessing.rs rename to stdlib/src/multiprocessing.rs index a03643ea1d..f508f9a5d5 100644 --- a/vm/src/stdlib/multiprocessing.rs +++ b/stdlib/src/multiprocessing.rs @@ -3,7 +3,7 @@ pub(crate) use _multiprocessing::make_module; #[cfg(windows)] #[pymodule] mod _multiprocessing { - use crate::{function::ArgBytesLike, stdlib::os, PyResult, VirtualMachine}; + use crate::vm::{function::ArgBytesLike, stdlib::os, PyResult, VirtualMachine}; use winapi::um::winsock2::{self, SOCKET}; #[pyfunction] diff --git a/vm/src/stdlib/platform.rs b/stdlib/src/platform.rs similarity index 94% rename from vm/src/stdlib/platform.rs rename to stdlib/src/platform.rs index d834918de4..c700fc8358 100644 --- a/vm/src/stdlib/platform.rs +++ b/stdlib/src/platform.rs @@ -2,7 +2,7 @@ pub(crate) use decl::make_module; #[pymodule(name = "platform")] mod decl { - use crate::{version, VirtualMachine}; + use crate::vm::{version, VirtualMachine}; #[pyfunction] fn python_implementation(_vm: &VirtualMachine) -> String { diff --git a/vm/src/stdlib/posixsubprocess.rs b/stdlib/src/posixsubprocess.rs similarity index 98% rename from vm/src/stdlib/posixsubprocess.rs rename to stdlib/src/posixsubprocess.rs index 4409be5794..c30aa5304c 100644 --- a/vm/src/stdlib/posixsubprocess.rs +++ b/stdlib/src/posixsubprocess.rs @@ -1,4 +1,4 @@ -use crate::{ +use crate::vm::{ stdlib::{os::PyPathLike, posix}, {PyObjectRef, PyResult, PySequence, TryFromObject, VirtualMachine}, }; @@ -18,7 +18,7 @@ pub(crate) use _posixsubprocess::make_module; #[pymodule] mod _posixsubprocess { use super::{exec, CStrPathLike, ForkExecArgs, ProcArgs}; - use crate::{exceptions::IntoPyException, PyResult, VirtualMachine}; + use crate::vm::{exceptions::IntoPyException, PyResult, VirtualMachine}; #[pyfunction] fn fork_exec(args: ForkExecArgs, vm: &VirtualMachine) -> PyResult { diff --git a/vm/src/stdlib/pyexpat.rs b/stdlib/src/pyexpat.rs similarity index 98% rename from vm/src/stdlib/pyexpat.rs rename to stdlib/src/pyexpat.rs index 07dc0f5844..75672d3d14 100644 --- a/vm/src/stdlib/pyexpat.rs +++ b/stdlib/src/pyexpat.rs @@ -3,7 +3,7 @@ * */ -use crate::{PyObjectRef, VirtualMachine}; +use crate::vm::{extend_module, PyObjectRef, VirtualMachine}; pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { let module = _pyexpat::make_module(vm); @@ -31,14 +31,13 @@ macro_rules! create_property { #[pymodule(name = "pyexpat")] mod _pyexpat { - use crate::{ + use crate::vm::{ builtins::{PyStr, PyStrRef, PyTypeRef}, function::ArgBytesLike, function::{IntoFuncArgs, OptionalArg}, ItemProtocol, PyContext, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject, VirtualMachine, }; - use rustpython_common::lock::PyRwLock; use std::io::Cursor; use xml::reader::XmlEvent; diff --git a/vm/src/stdlib/random.rs b/stdlib/src/random.rs similarity index 99% rename from vm/src/stdlib/random.rs rename to stdlib/src/random.rs index 5269c12e0a..8f7cfc7fb2 100644 --- a/vm/src/stdlib/random.rs +++ b/stdlib/src/random.rs @@ -5,7 +5,7 @@ pub(crate) use _random::make_module; #[pymodule] mod _random { use crate::common::lock::PyMutex; - use crate::{ + use crate::vm::{ builtins::{PyIntRef, PyTypeRef}, function::OptionalOption, slots::SlotConstructor, diff --git a/vm/src/stdlib/re.rs b/stdlib/src/re.rs similarity index 100% rename from vm/src/stdlib/re.rs rename to stdlib/src/re.rs diff --git a/vm/src/stdlib/resource.rs b/stdlib/src/resource.rs similarity index 99% rename from vm/src/stdlib/resource.rs rename to stdlib/src/resource.rs index 1bac89d216..98098b181e 100644 --- a/vm/src/stdlib/resource.rs +++ b/stdlib/src/resource.rs @@ -2,7 +2,7 @@ pub(crate) use resource::make_module; #[pymodule] mod resource { - use crate::{ + use crate::vm::{ exceptions::IntoPyException, stdlib::os, IntoPyObject, PyObjectRef, PyResult, PyStructSequence, TryFromBorrowedObject, VirtualMachine, }; diff --git a/vm/src/stdlib/scproxy.rs b/stdlib/src/scproxy.rs similarity index 99% rename from vm/src/stdlib/scproxy.rs rename to stdlib/src/scproxy.rs index 770650af6f..f65c07133a 100644 --- a/vm/src/stdlib/scproxy.rs +++ b/stdlib/src/scproxy.rs @@ -4,7 +4,7 @@ pub(crate) use _scproxy::make_module; mod _scproxy { // straight-forward port of Modules/_scproxy.c - use crate::{ + use crate::vm::{ builtins::{PyDictRef, PyStr}, IntoPyObject, ItemProtocol, PyResult, VirtualMachine, }; diff --git a/vm/src/stdlib/select.rs b/stdlib/src/select.rs similarity index 98% rename from vm/src/stdlib/select.rs rename to stdlib/src/select.rs index 33a2a15e34..451b14affc 100644 --- a/vm/src/stdlib/select.rs +++ b/stdlib/src/select.rs @@ -1,13 +1,13 @@ -use crate::{PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine}; +use crate::vm::{PyObjectRef, PyResult, TryFromBorrowedObject, TryFromObject, VirtualMachine}; use std::{io, mem}; pub(crate) fn make_module(vm: &VirtualMachine) -> PyObjectRef { #[cfg(windows)] - super::nt::init_winsock(); + crate::vm::stdlib::nt::init_winsock(); #[cfg(unix)] { - use crate::PyClassImpl; + use crate::vm::PyClassImpl; decl::poll::PyPoll::make_class(&vm.ctx); } @@ -151,7 +151,7 @@ fn sec_to_timeval(sec: f64) -> timeval { #[pymodule(name = "select")] mod decl { use super::*; - use crate::{ + use crate::vm::{ exceptions::IntoPyException, function::OptionalOption, stdlib::time, utils::Either, PyObjectRef, PyResult, VirtualMachine, }; @@ -252,7 +252,7 @@ mod decl { #[cfg(unix)] pub(super) mod poll { use super::*; - use crate::{ + use crate::vm::{ builtins::PyFloat, common::lock::PyMutex, function::OptionalArg, stdlib::io::Fildes, IntoPyObject, PyValue, TypeProtocol, }; diff --git a/vm/src/stdlib/socket.rs b/stdlib/src/socket.rs similarity index 98% rename from vm/src/stdlib/socket.rs rename to stdlib/src/socket.rs index 63b8778960..f509f69eaa 100644 --- a/vm/src/stdlib/socket.rs +++ b/stdlib/src/socket.rs @@ -1,8 +1,11 @@ use crate::common::lock::{PyMappedRwLockReadGuard, PyRwLock, PyRwLockReadGuard}; -use crate::{ +use crate::vm::{ builtins::{PyBaseExceptionRef, PyStrRef, PyTupleRef, PyTypeRef}, exceptions::IntoPyException, - function::{ArgBytesLike, ArgMemoryBuffer, FuncArgs, OptionalArg, OptionalOption}, + extend_module, + function::{ArgBytesLike, ArgMemoryBuffer}, + function::{FuncArgs, OptionalArg, OptionalOption}, + named_function, py_module, utils::{Either, ToCString}, IntoPyObject, PyClassImpl, PyObjectRef, PyResult, PyValue, TryFromBorrowedObject, TryFromObject, TypeProtocol, VirtualMachine, @@ -301,7 +304,7 @@ impl PySocket { #[cfg(unix)] c::AF_UNIX => { use std::os::unix::ffi::OsStrExt; - let buf = crate::function::ArgStrOrBytesLike::try_from_object(vm, addr)?; + let buf = crate::vm::function::ArgStrOrBytesLike::try_from_object(vm, addr)?; let path = &*buf.borrow_bytes(); if cfg!(any(target_os = "linux", target_os = "android")) && path.first() == Some(&0) { @@ -506,7 +509,7 @@ impl PySocket { ) }; if res < 0 { - return Err(super::os::errno_err(vm)); + return Err(crate::vm::stdlib::os::errno_err(vm)); } } cfg_if::cfg_if! { @@ -835,7 +838,7 @@ impl PySocket { ) }; if ret < 0 { - Err(super::os::errno_err(vm)) + Err(crate::vm::stdlib::os::errno_err(vm)) } else { Ok(vm.ctx.new_int(flag)) } @@ -849,7 +852,7 @@ impl PySocket { unsafe { c::getsockopt(fd, level, name, buf.as_mut_ptr() as *mut _, &mut buflen) }; buf.truncate(buflen as usize); if ret < 0 { - Err(super::os::errno_err(vm)) + Err(crate::vm::stdlib::os::errno_err(vm)) } else { Ok(vm.ctx.new_bytes(buf)) } @@ -889,7 +892,7 @@ impl PySocket { } }; if ret < 0 { - Err(super::os::errno_err(vm)) + Err(crate::vm::stdlib::os::errno_err(vm)) } else { Ok(()) } @@ -1190,7 +1193,7 @@ pub(super) fn sock_select( } #[cfg(windows)] { - use crate::stdlib::select; + use crate::select; let mut reads = select::FdSet::new(); let mut writes = select::FdSet::new(); @@ -1418,7 +1421,7 @@ type IfIndex = winapi::shared::ifdef::NET_IFINDEX; #[cfg(not(target_os = "redox"))] fn _socket_if_nametoindex(name: PyObjectRef, vm: &VirtualMachine) -> PyResult { - let name = super::os::FsPath::try_from(name, true, vm)?; + let name = crate::vm::stdlib::os::FsPath::try_from(name, true, vm)?; let name = ffi::CString::new(name.as_bytes()).map_err(|err| err.into_pyexception(vm))?; let ret = unsafe { c::if_nametoindex(name.as_ptr()) }; @@ -1435,7 +1438,7 @@ fn _socket_if_indextoname(index: IfIndex, vm: &VirtualMachine) -> PyResult PyResult { let newsock = sock.try_clone().map_err(|e| e.into_pyexception(vm))?; let fd = into_sock_fileno(newsock); #[cfg(windows)] - super::nt::raw_set_handle_inheritable(fd as _, false).map_err(|e| e.into_pyexception(vm))?; + crate::vm::stdlib::nt::raw_set_handle_inheritable(fd as _, false) + .map_err(|e| e.into_pyexception(vm))?; Ok(fd) } @@ -1798,7 +1802,7 @@ fn close_inner(x: RawSocket, vm: &VirtualMachine) -> PyResult<()> { use winapi::um::winsock2::closesocket as close; let ret = unsafe { close(x as _) }; if ret < 0 { - let err = super::os::errno(); + let err = crate::vm::stdlib::os::errno(); if err.raw_os_error() != Some(errcode!(ECONNRESET)) { return Err(err.into_pyexception(vm)); } @@ -1819,7 +1823,7 @@ rustpython_common::static_cell! { pub fn make_module(vm: &VirtualMachine) -> PyObjectRef { #[cfg(windows)] - super::nt::init_winsock(); + crate::vm::stdlib::nt::init_winsock(); let ctx = &vm.ctx; let socket_timeout = TIMEOUT_ERROR diff --git a/vm/src/stdlib/ssl.rs b/stdlib/src/ssl.rs similarity index 99% rename from vm/src/stdlib/ssl.rs rename to stdlib/src/ssl.rs index 360a996178..f97791f6b4 100644 --- a/vm/src/stdlib/ssl.rs +++ b/stdlib/src/ssl.rs @@ -3,10 +3,12 @@ use crate::common::{ ascii, lock::{PyRwLock, PyRwLockWriteGuard}, }; -use crate::{ +use crate::vm::{ builtins::{PyBaseException, PyBaseExceptionRef, PyStrRef, PyType, PyTypeRef, PyWeak}, exceptions::{self, IntoPyException}, + extend_module, function::{ArgBytesLike, ArgCallable, ArgMemoryBuffer, ArgStrOrBytesLike, OptionalArg}, + named_function, py_module, slots::SlotConstructor, stdlib::os::PyPathLike, types::create_simple_type, @@ -139,9 +141,10 @@ fn obj2py(obj: &Asn1ObjectRef) -> PyNid { #[cfg(windows)] fn _ssl_enum_certificates(store_name: PyStrRef, vm: &VirtualMachine) -> PyResult { - use crate::builtins::set::PyFrozenSet; + use crate::vm::builtins::PyFrozenSet; use schannel::{cert_context::ValidUses, cert_store::CertStore, RawPointer}; use winapi::um::wincrypt; + // TODO: check every store for it, not just 2 of them: // https://github.com/python/cpython/blob/3.8/Modules/_ssl.c#L5603-L5610 let open_fns = [CertStore::open_current_user, CertStore::open_local_machine]; @@ -495,7 +498,7 @@ impl PySslContext { if ret != 1 { let errno = std::io::Error::last_os_error().raw_os_error().unwrap(); let err = if errno != 0 { - super::os::errno_err(vm) + crate::vm::stdlib::os::errno_err(vm) } else { convert_openssl_error(vm, ErrorStack::get()) }; diff --git a/vm/src/stdlib/syslog.rs b/stdlib/src/syslog.rs similarity index 99% rename from vm/src/stdlib/syslog.rs rename to stdlib/src/syslog.rs index a8631f6ba9..3429924af9 100644 --- a/vm/src/stdlib/syslog.rs +++ b/stdlib/src/syslog.rs @@ -3,7 +3,7 @@ pub(crate) use syslog::make_module; #[pymodule(name = "syslog")] mod syslog { use crate::common::lock::PyRwLock; - use crate::{ + use crate::vm::{ builtins::{PyStr, PyStrRef}, function::{OptionalArg, OptionalOption}, utils::ToCString, diff --git a/vm/src/stdlib/termios.rs b/stdlib/src/termios.rs similarity index 99% rename from vm/src/stdlib/termios.rs rename to stdlib/src/termios.rs index acc9b76b50..65d8eb830e 100644 --- a/vm/src/stdlib/termios.rs +++ b/stdlib/src/termios.rs @@ -2,7 +2,7 @@ pub(crate) use self::termios::make_module; #[pymodule] mod termios { - use crate::{ + use crate::vm::{ builtins::PyBaseExceptionRef, builtins::{PyBytes, PyInt, PyListRef, PyTypeRef}, IntoPyObject, PyObjectRef, PyResult, TryFromObject, VirtualMachine, diff --git a/vm/src/stdlib/unicodedata.rs b/stdlib/src/unicodedata.rs similarity index 96% rename from vm/src/stdlib/unicodedata.rs rename to stdlib/src/unicodedata.rs index b75e943a7e..c5417e9c7b 100644 --- a/vm/src/stdlib/unicodedata.rs +++ b/stdlib/src/unicodedata.rs @@ -2,9 +2,9 @@ See also: https://docs.python.org/3/library/unicodedata.html */ -use crate::{ - builtins::PyStrRef, function::OptionalArg, PyClassImpl, PyObject, PyObjectRef, PyResult, - PyValue, VirtualMachine, +use crate::vm::{ + builtins::PyStrRef, extend_module, function::OptionalArg, py_module, PyClassImpl, PyObject, + PyObjectRef, PyResult, PyValue, VirtualMachine, }; use itertools::Itertools; use unic_char_property::EnumeratedCharProperty; diff --git a/vm/src/stdlib/zlib.rs b/stdlib/src/zlib.rs similarity index 99% rename from vm/src/stdlib/zlib.rs rename to stdlib/src/zlib.rs index 77ff599650..fb5f163cba 100644 --- a/vm/src/stdlib/zlib.rs +++ b/stdlib/src/zlib.rs @@ -3,7 +3,7 @@ pub(crate) use decl::make_module; #[pymodule(name = "zlib")] mod decl { use crate::common::lock::PyMutex; - use crate::{ + use crate::vm::{ builtins::{PyBaseExceptionRef, PyBytes, PyBytesRef, PyIntRef, PyTypeRef}, function::{ArgBytesLike, OptionalArg}, types::create_simple_type, diff --git a/vm/Cargo.toml b/vm/Cargo.toml index d1851f1fab..e778579693 100644 --- a/vm/Cargo.toml +++ b/vm/Cargo.toml @@ -10,8 +10,6 @@ include = ["src/**/*.rs", "Cargo.toml", "build.rs", "Lib/**/*.py"] [features] default = ["compile-parse", "threading", "pylib"] -# TODO: use resolver = "2" instead of features -zlib = ["libz-sys", "flate2/zlib"] vm-tracing-logging = [] flame-it = ["flame", "flamer"] pylib = ["rustpython-pylib"] @@ -23,28 +21,13 @@ ast = ["rustpython-ast"] compiler = ["rustpython-compiler", "rustpython-compiler-core", "ast"] parser = ["rustpython-parser", "ast"] -ssl = ["openssl", "openssl-sys", "foreign-types-shared"] -ssl-vendor = ["ssl", "openssl/vendored", "openssl-probe"] - [dependencies] -# Crypto: -digest = "0.9.0" -md-5 = "0.9.1" -sha-1 = "0.9.4" -sha2 = "0.9.3" -sha3 = "0.9.1" -blake2 = "0.9.1" -xml-rs = "0.8" - num-complex = { version = "0.4.0", features = ["serde"] } num-bigint = { version = "0.4.2", features = ["serde"] } num-traits = "0.2.8" num-integer = "0.1.41" num-rational = "0.4.0" -rand = "0.8" -rand_core = "0.6" -getrandom = { version = "0.2", features = ["js"] } -log = "0.4" +rustpython-common = { path = "../common" } rustpython-derive = { path = "../derive", version = "0.1.2" } rustpython-ast = { path = "../ast", optional = true, version = "0.1" } rustpython-parser = { path = "../parser", optional = true, version = "0.1.2" } @@ -53,8 +36,10 @@ rustpython-compiler-core = { path = "../compiler", optional = true, version = "0 rustpython-bytecode = { path = "../bytecode", version = "0.1.2" } rustpython-jit = { path = "../jit", optional = true, version = "0.1.2" } rustpython-pylib = { path = "pylib-crate", optional = true, version = "0.1.0" } +rand = "0.8" +getrandom = { version = "0.2", features = ["js"] } +log = "0.4" serde = { version = "1.0.66", features = ["derive"] } -puruspe = "0.1" caseless = "0.2.1" chrono = { version = "0.4", features = ["wasmbind"] } lexical-core = "0.7" @@ -63,13 +48,10 @@ hex = "0.4.0" hexf-parse = "0.1.0" indexmap = "1.0.2" ahash = "0.7.2" -crc = "^1.0.0" bitflags = "1.2.1" libc = "0.2.99" nix = "0.22.1" -csv-core = "0.1" paste = "1.0.5" -base64 = "0.13" is-macro = "0.1" result-like = "^0.3" num_enum = "0.5" @@ -87,11 +69,9 @@ memchr = "2" crc32fast = "1.2.0" adler32 = "1.0.3" flate2 = "1.0.20" -libz-sys = { version = "1.0", optional = true } once_cell = "1" # RustPython crates implementing functionality based on CPython -mt19937 = "2.0" sre-engine = "0.1.2" # to work on sre-engine locally # sre-engine = { path = "../../sre-engine" } @@ -102,17 +82,13 @@ unicode_names2 = "0.4" # https://github.com/RustPython/RustPython/pull/832#discussion_r275428939 unicode-casing = "0.1" # update version all at the same time -unic-char-property = "0.9" -unic-normal = "0.9" unic-ucd-bidi = "0.9" unic-ucd-category = "0.9" -unic-ucd-age = "0.9" unic-ucd-ident = "0.9" flame = { version = "0.2", optional = true } flamer = { version = "0.4", optional = true } -rustpython-common = { path = "../common" } ascii = "1.0.0" [target.'cfg(unix)'.dependencies] @@ -122,28 +98,14 @@ strum = "0.21" strum_macros = "0.21" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -gethostname = "0.2.0" -socket2 = { version = "0.4.0", features = ["all"] } -dns-lookup = "1.0" rustyline = "9" -openssl = { version = "0.10.32", optional = true } -openssl-sys = { version = "0.9", optional = true } -openssl-probe = { version = "0.1", optional = true } which = "4.0" -foreign-types-shared = { version = "0.1", optional = true } [target.'cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))'.dependencies] num_cpus = "1" -[target.'cfg(all(unix, not(target_os = "redox")))'.dependencies] -termios = "0.3" - -[target.'cfg(target_os = "macos")'.dependencies] -system-configuration = "0.4" - [target.'cfg(windows)'.dependencies] winreg = "0.8.0" -schannel = "0.1" widestring = "0.4" [target.'cfg(windows)'.dependencies.winapi] diff --git a/vm/build.rs b/vm/build.rs index c1daf2eb44..5566822661 100644 --- a/vm/build.rs +++ b/vm/build.rs @@ -31,33 +31,6 @@ fn main() { std::env::vars_os().format_with(", ", |(k, v), f| f(&format_args!("{:?} => {:?}", k, v))) ) .unwrap(); - - #[allow(clippy::unusual_byte_groupings)] - if let Ok(v) = env::var("DEP_OPENSSL_VERSION_NUMBER") { - println!("cargo:rustc-env=OPENSSL_API_VERSION={}", v); - // cfg setup from openssl crate's build script - let version = u64::from_str_radix(&v, 16).unwrap(); - if version >= 0x1_00_01_00_0 { - println!("cargo:rustc-cfg=ossl101"); - } - if version >= 0x1_00_02_00_0 { - println!("cargo:rustc-cfg=ossl102"); - } - if version >= 0x1_01_00_00_0 { - println!("cargo:rustc-cfg=ossl110"); - } - if version >= 0x1_01_00_07_0 { - println!("cargo:rustc-cfg=ossl110g"); - } - if version >= 0x1_01_01_00_0 { - println!("cargo:rustc-cfg=ossl111"); - } - } - if let Ok(v) = env::var("DEP_OPENSSL_CONF") { - for conf in v.split(',') { - println!("cargo:rustc-cfg=osslconf=\"{}\"", conf); - } - } } fn git_hash() -> String { diff --git a/vm/src/builtins/set.rs b/vm/src/builtins/set.rs index 7ce164d7a7..7a6510b22f 100644 --- a/vm/src/builtins/set.rs +++ b/vm/src/builtins/set.rs @@ -639,7 +639,7 @@ impl SlotConstructor for PyFrozenSet { #[pyimpl(flags(BASETYPE), with(Hashable, Comparable, Iterable, SlotConstructor))] impl PyFrozenSet { // Also used by ssl.rs windows. - pub(crate) fn from_iter( + pub fn from_iter( vm: &VirtualMachine, it: impl IntoIterator, ) -> PyResult { diff --git a/vm/src/exceptions.rs b/vm/src/exceptions.rs index 1ba18fe0d6..645875626e 100644 --- a/vm/src/exceptions.rs +++ b/vm/src/exceptions.rs @@ -934,7 +934,7 @@ impl serde::Serialize for SerializeException<'_> { } } -pub(crate) fn cstring_error(vm: &VirtualMachine) -> PyBaseExceptionRef { +pub fn cstring_error(vm: &VirtualMachine) -> PyBaseExceptionRef { vm.new_value_error("embedded null character".to_owned()) } diff --git a/vm/src/import.rs b/vm/src/import.rs index 4af056e98a..77a43b13d2 100644 --- a/vm/src/import.rs +++ b/vm/src/import.rs @@ -89,7 +89,7 @@ pub fn import_frozen(vm: &VirtualMachine, module_name: &str) -> PyResult { pub fn import_builtin(vm: &VirtualMachine, module_name: &str) -> PyResult { vm.state - .stdlib_inits + .module_inits .get(module_name) .ok_or_else(|| { vm.new_import_error( diff --git a/vm/src/protocol/buffer.rs b/vm/src/protocol/buffer.rs index 9b2e3964c6..58137b55f8 100644 --- a/vm/src/protocol/buffer.rs +++ b/vm/src/protocol/buffer.rs @@ -133,7 +133,7 @@ impl Clone for PyBuffer { } } -pub(crate) trait ResizeGuard<'a> { +pub trait ResizeGuard<'a> { type Resizable: 'a; fn try_resizable(&'a self, vm: &VirtualMachine) -> PyResult; } diff --git a/vm/src/protocol/mod.rs b/vm/src/protocol/mod.rs index ce4906da5e..24692be4c7 100644 --- a/vm/src/protocol/mod.rs +++ b/vm/src/protocol/mod.rs @@ -1,5 +1,5 @@ mod buffer; mod iter; -pub(crate) use buffer::{BufferInternal, BufferOptions, PyBuffer, ResizeGuard}; +pub use buffer::{BufferInternal, BufferOptions, PyBuffer, ResizeGuard}; pub use iter::PyIter; diff --git a/vm/src/stdlib/imp.rs b/vm/src/stdlib/imp.rs index 2d070c06a1..c147fcb84c 100644 --- a/vm/src/stdlib/imp.rs +++ b/vm/src/stdlib/imp.rs @@ -46,7 +46,7 @@ fn _imp_extension_suffixes(vm: &VirtualMachine) -> PyResult { } fn _imp_is_builtin(name: PyStrRef, vm: &VirtualMachine) -> bool { - vm.state.stdlib_inits.contains_key(name.as_str()) + vm.state.module_inits.contains_key(name.as_str()) } fn _imp_is_frozen(name: PyStrRef, vm: &VirtualMachine) -> bool { @@ -60,7 +60,7 @@ fn _imp_create_builtin(spec: PyObjectRef, vm: &VirtualMachine) -> PyResult { if let Ok(module) = sys_modules.get_item(name.clone(), vm) { Ok(module) - } else if let Some(make_module_func) = vm.state.stdlib_inits.get(name.as_ref()) { + } else if let Some(make_module_func) = vm.state.module_inits.get(name.as_ref()) { Ok(make_module_func(vm)) } else { Ok(vm.ctx.none()) diff --git a/vm/src/stdlib/mod.rs b/vm/src/stdlib/mod.rs index f2d73cc6b8..349b9f2a10 100644 --- a/vm/src/stdlib/mod.rs +++ b/vm/src/stdlib/mod.rs @@ -1,90 +1,51 @@ -mod array; #[cfg(feature = "rustpython-ast")] pub(crate) mod ast; mod atexit; -mod binascii; -mod bisect; pub mod builtins; -mod cmath; mod codecs; mod collections; -mod csv; -mod dis; mod errno; mod functools; -mod hashlib; mod imp; -pub(crate) mod io; +pub mod io; mod itertools; -mod json; -#[cfg(feature = "rustpython-parser")] -mod keyword; mod marshal; -mod math; mod operator; -mod platform; -mod pyexpat; pub(crate) mod pystruct; -mod random; // TODO: maybe make this an extension module, if we ever get those // mod re; -#[cfg(not(target_arch = "wasm32"))] -mod socket; mod sre; mod string; #[cfg(feature = "rustpython-compiler")] mod symtable; mod sysconfigdata; -#[cfg(unix)] -mod syslog; #[cfg(feature = "threading")] mod thread; -mod time; -mod unicodedata; +pub mod time; mod warnings; mod weakref; -mod zlib; #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] #[macro_use] pub mod os; #[cfg(windows)] -pub(crate) mod nt; +pub mod nt; #[cfg(unix)] -pub(crate) mod posix; +pub mod posix; #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] #[cfg(not(any(unix, windows)))] pub(crate) mod posix_compat; #[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))] #[cfg(not(any(unix, windows)))] -pub(crate) use posix_compat as posix; +pub use posix_compat as posix; -#[cfg(not(target_arch = "wasm32"))] -mod faulthandler; -#[cfg(any(unix, target_os = "wasi"))] -mod fcntl; #[cfg(windows)] pub(crate) mod msvcrt; -#[cfg(not(target_arch = "wasm32"))] -mod multiprocessing; -#[cfg(unix)] -mod posixsubprocess; #[cfg(all(unix, not(any(target_os = "android", target_os = "redox"))))] mod pwd; -// libc is missing constants on redox -#[cfg(all(unix, not(target_os = "redox")))] -mod resource; -#[cfg(target_os = "macos")] -mod scproxy; -#[cfg(not(target_arch = "wasm32"))] -mod select; #[cfg(not(target_arch = "wasm32"))] pub(crate) mod signal; -#[cfg(all(not(target_arch = "wasm32"), feature = "ssl"))] -mod ssl; pub mod sys; -#[cfg(all(unix, not(target_os = "redox")))] -mod termios; #[cfg(windows)] mod winapi; #[cfg(windows)] @@ -118,36 +79,22 @@ pub fn get_module_inits() -> StdlibMap { modules! { #[cfg(all())] { - "array" => array::make_module, "atexit" => atexit::make_module, - "binascii" => binascii::make_module, - "_bisect" => bisect::make_module, - "cmath" => cmath::make_module, "_codecs" => codecs::make_module, "_collections" => collections::make_module, - "_csv" => csv::make_module, - "dis" => dis::make_module, "errno" => errno::make_module, "_functools" => functools::make_module, - "hashlib" => hashlib::make_module, "itertools" => itertools::make_module, "_io" => io::make_module, - "_json" => json::make_module, "marshal" => marshal::make_module, - "math" => math::make_module, "_operator" => operator::make_module, - "pyexpat" => pyexpat::make_module, - "_platform" => platform::make_module, - "_random" => random::make_module, "_sre" => sre::make_module, "_string" => string::make_module, "_struct" => pystruct::make_module, "time" => time::make_module, "_weakref" => weakref::make_module, "_imp" => imp::make_module, - "unicodedata" => unicodedata::make_module, "_warnings" => warnings::make_module, - "zlib" => zlib::make_module, sys::sysconfigdata_name() => sysconfigdata::make_module, } // parser related modules: @@ -155,10 +102,6 @@ pub fn get_module_inits() -> StdlibMap { { "_ast" => ast::make_module, } - #[cfg(feature = "rustpython-parser")] - { - "keyword" => keyword::make_module, - } // compiler related modules: #[cfg(feature = "rustpython-compiler")] { @@ -167,20 +110,12 @@ pub fn get_module_inits() -> StdlibMap { #[cfg(any(unix, target_os = "wasi"))] { "posix" => posix::make_module, - "fcntl" => fcntl::make_module, + // "fcntl" => fcntl::make_module, } // disable some modules on WASM #[cfg(not(target_arch = "wasm32"))] { - "_socket" => socket::make_module, - "_multiprocessing" => multiprocessing::make_module, "_signal" => signal::make_module, - "select" => select::make_module, - "faulthandler" => faulthandler::make_module, - } - #[cfg(feature = "ssl")] - { - "_ssl" => ssl::make_module, } #[cfg(all(feature = "threading", not(target_arch = "wasm32")))] { @@ -191,16 +126,6 @@ pub fn get_module_inits() -> StdlibMap { { "pwd" => pwd::make_module, } - #[cfg(all(unix, not(target_os = "redox")))] - { - "termios" => termios::make_module, - "resource" => resource::make_module, - } - #[cfg(unix)] - { - "_posixsubprocess" => posixsubprocess::make_module, - "syslog" => syslog::make_module, - } // Windows-only #[cfg(windows)] { @@ -209,9 +134,5 @@ pub fn get_module_inits() -> StdlibMap { "_winapi" => winapi::make_module, "winreg" => winreg::make_module, } - #[cfg(target_os = "macos")] - { - "_scproxy" => scproxy::make_module, - } } } diff --git a/vm/src/stdlib/nt.rs b/vm/src/stdlib/nt.rs index 3c08219d81..62267e3266 100644 --- a/vm/src/stdlib/nt.rs +++ b/vm/src/stdlib/nt.rs @@ -1,6 +1,6 @@ use crate::{PyObjectRef, VirtualMachine}; -pub(crate) use module::raw_set_handle_inheritable; +pub use module::raw_set_handle_inheritable; pub(crate) fn make_module(vm: &VirtualMachine) -> PyObjectRef { let module = module::make_module(vm); @@ -368,10 +368,7 @@ pub(crate) mod module { } } - pub(crate) fn raw_set_handle_inheritable( - handle: intptr_t, - inheritable: bool, - ) -> io::Result<()> { + pub fn raw_set_handle_inheritable(handle: intptr_t, inheritable: bool) -> io::Result<()> { use um::winbase::HANDLE_FLAG_INHERIT; let flags = if inheritable { HANDLE_FLAG_INHERIT } else { 0 }; let res = diff --git a/vm/src/stdlib/sys.rs b/vm/src/stdlib/sys.rs index fda309a85d..fa6480c920 100644 --- a/vm/src/stdlib/sys.rs +++ b/vm/src/stdlib/sys.rs @@ -637,7 +637,7 @@ setprofile() -- set the global profiling function setrecursionlimit() -- set the max recursion depth for the interpreter settrace() -- set the global debug tracing function "; - let mut module_names: Vec<_> = vm.state.stdlib_inits.keys().cloned().collect(); + let mut module_names: Vec<_> = vm.state.module_inits.keys().cloned().collect(); module_names.push("sys".into()); module_names.push("builtins".into()); module_names.sort(); diff --git a/vm/src/vm.rs b/vm/src/vm.rs index 9aff2a72e0..5e2a00614e 100644 --- a/vm/src/vm.rs +++ b/vm/src/vm.rs @@ -117,7 +117,7 @@ pub(crate) mod thread { pub struct PyGlobalState { pub settings: PySettings, - pub stdlib_inits: stdlib::StdlibMap, + pub module_inits: stdlib::StdlibMap, pub frozen: HashMap, pub stacksize: AtomicCell, pub thread_count: AtomicCell, @@ -263,7 +263,7 @@ impl VirtualMachine { const NONE: Option = None; let signal_handlers = RefCell::new([NONE; NSIG]); - let stdlib_inits = stdlib::get_module_inits(); + let module_inits = stdlib::get_module_inits(); let hash_secret = match settings.hash_seed { Some(seed) => HashSecret::new(seed), @@ -288,7 +288,7 @@ impl VirtualMachine { repr_guards: RefCell::default(), state: PyRc::new(PyGlobalState { settings, - stdlib_inits, + module_inits, frozen: HashMap::default(), stacksize: AtomicCell::new(0), thread_count: AtomicCell::new(0), @@ -391,7 +391,7 @@ impl VirtualMachine { { let state = PyRc::get_mut(&mut self.state) .expect("can't add_native_module when there are multiple threads"); - state.stdlib_inits.insert(name.into(), module); + state.module_inits.insert(name.into(), module); } /// Can only be used in the initialization closure passed to [`Interpreter::new_with_init`]