Skip to content

Commit 559d91a

Browse files
youknowoneyouknowone
authored
Merge pull request #1926 from palaviv/threading-stdlib-2
Convert more stdlib objects to thread safe
2 parents 5877f41 + cc2e97c commit 559d91a

File tree

7 files changed

+92
-36
lines changed

7 files changed

+92
-36
lines changed

‎Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vm/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ num_enum = "0.4"
7070
smallbox = "0.8"
7171
bstr = "0.2.12"
7272
crossbeam-utils = "0.7"
73+
generational-arena = "0.2"
7374

7475
## unicode stuff
7576
unicode_names2 = "0.4"

‎vm/src/stdlib/json.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
use crate::obj::objiter;
22
use crate::obj::objstr::PyStringRef;
33
use crate::obj::{objbool, objtype::PyClassRef};
4-
use crate::pyobject::{IdProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue};
4+
use crate::pyobject::{IdProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, ThreadSafe};
55
use crate::VirtualMachine;
66

77
use num_bigint::BigInt;
@@ -19,6 +19,8 @@ struct JsonScanner {
1919
ctx: PyObjectRef,
2020
}
2121

22+
impl ThreadSafe for JsonScanner {}
23+
2224
impl PyValue for JsonScanner {
2325
fn class(vm: &VirtualMachine) -> PyClassRef {
2426
vm.class("_json", "make_scanner")

‎vm/src/stdlib/os.rs

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
use std::cell::{Cell, RefCell};
21
use std::ffi;
32
use std::fs::File;
43
use std::fs::OpenOptions;
@@ -7,10 +6,12 @@ use std::io::{self, ErrorKind, Read, Write};
76
use std::os::unix::fs::OpenOptionsExt;
87
#[cfg(windows)]
98
use std::os::windows::fs::OpenOptionsExt;
9+
use std::sync::RwLock;
1010
use std::time::{Duration, SystemTime};
1111
use std::{env, fs};
1212

1313
use bitflags::bitflags;
14+
use crossbeam_utils::atomic::AtomicCell;
1415
#[cfg(unix)]
1516
use nix::errno::Errno;
1617
#[cfg(all(unix, not(target_os = "redox")))]
@@ -35,8 +36,8 @@ use crate::obj::objstr::{PyString, PyStringRef};
3536
use crate::obj::objtuple::PyTupleRef;
3637
use crate::obj::objtype::PyClassRef;
3738
use crate::pyobject::{
38-
Either, ItemProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject,
39-
TypeProtocol,
39+
Either, ItemProtocol, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, ThreadSafe,
40+
TryFromObject, TypeProtocol,
4041
};
4142
use crate::vm::VirtualMachine;
4243

@@ -597,6 +598,8 @@ struct DirEntry {
597598
mode: OutputMode,
598599
}
599600

601+
impl ThreadSafe for DirEntry {}
602+
600603
type DirEntryRef = PyRef<DirEntry>;
601604

602605
impl PyValue for DirEntry {
@@ -683,11 +686,13 @@ impl DirEntryRef {
683686
#[pyclass]
684687
#[derive(Debug)]
685688
struct ScandirIterator {
686-
entries: RefCell<fs::ReadDir>,
687-
exhausted: Cell<bool>,
689+
entries: RwLock<fs::ReadDir>,
690+
exhausted: AtomicCell<bool>,
688691
mode: OutputMode,
689692
}
690693

694+
impl ThreadSafe for ScandirIterator {}
695+
691696
impl PyValue for ScandirIterator {
692697
fn class(vm: &VirtualMachine) -> PyClassRef {
693698
vm.class(MODULE_NAME, "ScandirIter")
@@ -698,11 +703,11 @@ impl PyValue for ScandirIterator {
698703
impl ScandirIterator {
699704
#[pymethod(name = "__next__")]
700705
fn next(&self, vm: &VirtualMachine) -> PyResult {
701-
if self.exhausted.get() {
706+
if self.exhausted.load() {
702707
return Err(objiter::new_stop_iteration(vm));
703708
}
704709

705-
match self.entries.borrow_mut().next() {
710+
match self.entries.write().unwrap().next() {
706711
Some(entry) => match entry {
707712
Ok(entry) => Ok(DirEntry {
708713
entry,
@@ -713,15 +718,15 @@ impl ScandirIterator {
713718
Err(s) => Err(convert_io_error(vm, s)),
714719
},
715720
None => {
716-
self.exhausted.set(true);
721+
self.exhausted.store(true);
717722
Err(objiter::new_stop_iteration(vm))
718723
}
719724
}
720725
}
721726

722727
#[pymethod]
723728
fn close(&self) {
724-
self.exhausted.set(true);
729+
self.exhausted.store(true);
725730
}
726731

727732
#[pymethod(name = "__iter__")]
@@ -748,8 +753,8 @@ fn os_scandir(path: OptionalArg<PyPathLike>, vm: &VirtualMachine) -> PyResult {
748753

749754
match fs::read_dir(path.path) {
750755
Ok(iter) => Ok(ScandirIterator {
751-
entries: RefCell::new(iter),
752-
exhausted: Cell::new(false),
756+
entries: RwLock::new(iter),
757+
exhausted: AtomicCell::new(false),
753758
mode: path.mode,
754759
}
755760
.into_ref(vm)

‎vm/src/stdlib/pystruct.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ mod _struct {
2727
objtuple::PyTuple, objtype::PyClassRef,
2828
};
2929
use crate::pyobject::{
30-
Either, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, TryFromObject,
30+
Either, PyClassImpl, PyObjectRef, PyRef, PyResult, PyValue, ThreadSafe, TryFromObject,
3131
};
3232
use crate::VirtualMachine;
3333

@@ -679,6 +679,8 @@ mod _struct {
679679
fmt_str: PyStringRef,
680680
}
681681

682+
impl ThreadSafe for PyStruct {}
683+
682684
impl PyValue for PyStruct {
683685
fn class(vm: &VirtualMachine) -> PyClassRef {
684686
vm.class("_struct", "Struct")

‎vm/src/stdlib/random.rs

Lines changed: 63 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,9 @@ mod _random {
77
use crate::function::OptionalOption;
88
use crate::obj::objint::PyIntRef;
99
use crate::obj::objtype::PyClassRef;
10-
use crate::pyobject::{PyClassImpl, PyRef, PyResult, PyValue};
10+
use crate::pyobject::{PyClassImpl, PyRef, PyResult, PyValue, ThreadSafe};
1111
use crate::VirtualMachine;
12+
use generational_arena::{self, Arena};
1213
use num_bigint::{BigInt, Sign};
1314
use num_traits::Signed;
1415
use rand::RngCore;
@@ -53,12 +54,51 @@ mod _random {
5354
}
5455
}
5556

57+
thread_local!(static RNG_HANDLES: RefCell<Arena<PyRng>> = RefCell::new(Arena::new()));
58+
59+
#[derive(Debug)]
60+
struct RngHandle(generational_arena::Index);
61+
impl RngHandle {
62+
fn new(rng: PyRng) -> Self {
63+
let idx = RNG_HANDLES.with(|arena| arena.borrow_mut().insert(rng));
64+
RngHandle(idx)
65+
}
66+
fn exec<F, R>(&self, func: F) -> R
67+
where
68+
F: Fn(&mut PyRng) -> R,
69+
{
70+
RNG_HANDLES.with(|arena| {
71+
func(
72+
arena
73+
.borrow_mut()
74+
.get_mut(self.0)
75+
.expect("index was removed"),
76+
)
77+
})
78+
}
79+
fn replace(&self, rng: PyRng) {
80+
RNG_HANDLES.with(|arena| {
81+
*arena
82+
.borrow_mut()
83+
.get_mut(self.0)
84+
.expect("index was removed") = rng
85+
})
86+
}
87+
}
88+
impl Drop for RngHandle {
89+
fn drop(&mut self) {
90+
RNG_HANDLES.with(|arena| arena.borrow_mut().remove(self.0));
91+
}
92+
}
93+
5694
#[pyclass(name = "Random")]
5795
#[derive(Debug)]
5896
struct PyRandom {
59-
rng: RefCell<PyRng>,
97+
rng: RngHandle,
6098
}
6199

100+
impl ThreadSafe for PyRandom {}
101+
62102
impl PyValue for PyRandom {
63103
fn class(vm: &VirtualMachine) -> PyClassRef {
64104
vm.class("_random", "Random")
@@ -70,14 +110,14 @@ mod _random {
70110
#[pyslot(new)]
71111
fn new(cls: PyClassRef, vm: &VirtualMachine) -> PyResult<PyRef<Self>> {
72112
PyRandom {
73-
rng: RefCell::new(PyRng::default()),
113+
rng: RngHandle::new(PyRng::default()),
74114
}
75115
.into_ref_with_type(vm, cls)
76116
}
77117

78118
#[pymethod]
79119
fn random(&self) -> f64 {
80-
mt19937::gen_res53(&mut *self.rng.borrow_mut())
120+
self.rng.exec(mt19937::gen_res53)
81121
}
82122

83123
#[pymethod]
@@ -93,31 +133,32 @@ mod _random {
93133
}
94134
};
95135

96-
*self.rng.borrow_mut() = new_rng;
136+
self.rng.replace(new_rng);
97137
}
98138

99139
#[pymethod]
100-
fn getrandbits(&self, mut k: usize) -> BigInt {
101-
let mut rng = self.rng.borrow_mut();
140+
fn getrandbits(&self, k: usize) -> BigInt {
141+
self.rng.exec(|rng| {
142+
let mut k = k;
143+
let mut gen_u32 = |k| rng.next_u32() >> (32 - k) as u32;
102144

103-
let mut gen_u32 = |k| rng.next_u32() >> (32 - k) as u32;
104-
105-
if k <= 32 {
106-
return gen_u32(k).into();
107-
}
145+
if k <= 32 {
146+
return gen_u32(k).into();
147+
}
108148

109-
let words = (k - 1) / 8 + 1;
110-
let mut wordarray = vec![0u32; words];
149+
let words = (k - 1) / 8 + 1;
150+
let mut wordarray = vec![0u32; words];
111151

112-
let it = wordarray.iter_mut();
113-
#[cfg(target_endian = "big")]
114-
let it = it.rev();
115-
for word in it {
116-
*word = gen_u32(k);
117-
k -= 32;
118-
}
152+
let it = wordarray.iter_mut();
153+
#[cfg(target_endian = "big")]
154+
let it = it.rev();
155+
for word in it {
156+
*word = gen_u32(k);
157+
k -= 32;
158+
}
119159

120-
BigInt::from_slice(Sign::NoSign, &wordarray)
160+
BigInt::from_slice(Sign::NoSign, &wordarray)
161+
})
121162
}
122163
}
123164
}

‎vm/src/stdlib/re.rs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ use crate::function::{Args, OptionalArg};
1313
use crate::obj::objint::{PyInt, PyIntRef};
1414
use crate::obj::objstr::{PyString, PyStringRef};
1515
use crate::obj::objtype::PyClassRef;
16-
use crate::pyobject::{PyClassImpl, PyObjectRef, PyResult, PyValue, TryFromObject};
16+
use crate::pyobject::{PyClassImpl, PyObjectRef, PyResult, PyValue, ThreadSafe, TryFromObject};
1717
use crate::vm::VirtualMachine;
1818

1919
#[pyclass(name = "Pattern")]
@@ -23,6 +23,8 @@ struct PyPattern {
2323
pattern: String,
2424
}
2525

26+
impl ThreadSafe for PyPattern {}
27+
2628
const IGNORECASE: usize = 2;
2729
const LOCALE: usize = 4;
2830
const MULTILINE: usize = 8;
@@ -75,6 +77,8 @@ struct PyMatch {
7577
captures: Vec<Option<(usize, usize)>>,
7678
}
7779

80+
impl ThreadSafe for PyMatch {}
81+
7882
impl fmt::Debug for PyMatch {
7983
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
8084
write!(f, "Match()")

0 commit comments

Comments
 (0)