Skip to content

Reimpl Buffer Protocol and memoryview support ndarray with shape, stride and suboffset #3340

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 20 commits into from
Nov 13, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions Lib/test/test_gzip.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,6 @@ def test_write_read_with_pathlike_file(self):
self.assertEqual(d, data1 * 51)
self.assertIsInstance(f.name, str)

# TODO: RUSTPYTHON
@unittest.expectedFailure
# The following test_write_xy methods test that write accepts
# the corresponding bytes-like object type as input
# and that the data written equals bytes(xy) in all cases.
Expand Down
2 changes: 0 additions & 2 deletions Lib/test/test_memoryview.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,6 @@ def setitem(value):
m = None
self.assertEqual(sys.getrefcount(b), oldrefcount)

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_setitem_writable(self):
if not self.rw_type:
self.skipTest("no writable type to test")
Expand Down
6 changes: 4 additions & 2 deletions extra_tests/snippets/memoryview.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ def test_slice():
assert m[::2][1:-1].tobytes() == b'357'
assert m[::2][::2].tobytes() == b'159'
assert m[::2][1::2].tobytes() == b'37'
assert m[::-1].tobytes() == b'987654321'
assert m[::-2].tobytes() == b'97531'

test_slice()

Expand All @@ -56,9 +58,9 @@ def test_resizable():
m.release()
b.append(6)
m2 = memoryview(b)
m4 = memoryview(b)
m4 = memoryview(m2)
assert_raises(BufferError, lambda: b.append(5))
m3 = memoryview(b)
m3 = memoryview(m2)
assert_raises(BufferError, lambda: b.append(5))
m2.release()
assert_raises(BufferError, lambda: b.append(5))
Expand Down
74 changes: 34 additions & 40 deletions stdlib/src/array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ pub(crate) use array::make_module;
#[pymodule(name = "array")]
mod array {
use crate::common::{
borrow::{BorrowedValue, BorrowedValueMut},
atomic::{self, AtomicUsize},
lock::{
PyMappedRwLockReadGuard, PyMappedRwLockWriteGuard, PyRwLock, PyRwLockReadGuard,
PyRwLockWriteGuard,
Expand All @@ -20,18 +20,17 @@ mod array {
ArgBytesLike, ArgIntoFloat, ArgIterable, IntoPyObject, IntoPyResult, OptionalArg,
},
protocol::{
BufferInternal, BufferOptions, BufferResizeGuard, PyBuffer, PyIterReturn,
BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn,
PyMappingMethods,
},
sliceable::{PySliceableSequence, PySliceableSequenceMut, SaturatedSlice, SequenceIndex},
types::{
AsBuffer, AsMapping, Comparable, Constructor, IterNext, IterNextIterable, Iterable,
PyComparisonOp,
},
IdProtocol, PyComparisonValue, PyObject, PyObjectRef, PyObjectView, PyRef, PyResult,
PyValue, TryFromObject, TypeProtocol, VirtualMachine,
IdProtocol, PyComparisonValue, PyObject, PyObjectRef, PyObjectView, PyObjectWrap, PyRef,
PyResult, PyValue, TryFromObject, TypeProtocol, VirtualMachine,
};
use crossbeam_utils::atomic::AtomicCell;
use itertools::Itertools;
use num_traits::ToPrimitive;
use std::cmp::Ordering;
Expand Down Expand Up @@ -615,7 +614,7 @@ mod array {
#[derive(Debug, PyValue)]
pub struct PyArray {
array: PyRwLock<ArrayContentType>,
exports: AtomicCell<usize>,
exports: AtomicUsize,
}

pub type PyArrayRef = PyRef<PyArray>;
Expand All @@ -624,7 +623,7 @@ mod array {
fn from(array: ArrayContentType) -> Self {
PyArray {
array: PyRwLock::new(array),
exports: AtomicCell::new(0),
exports: AtomicUsize::new(0),
}
}
}
Expand Down Expand Up @@ -1220,40 +1219,35 @@ mod array {
fn as_buffer(zelf: &PyObjectView<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
let array = zelf.read();
let buf = PyBuffer::new(
zelf.as_object().to_owned(),
PyArrayBufferInternal(zelf.to_owned()),
BufferOptions {
readonly: false,
len: array.len(),
itemsize: array.itemsize(),
format: array.typecode_str().into(),
..Default::default()
},
zelf.to_owned().into(),
BufferDescriptor::format(
array.len() * array.itemsize(),
false,
array.itemsize(),
array.typecode_str().into(),
),
&BUFFER_METHODS,
);
Ok(buf)
}
}

#[derive(Debug)]
struct PyArrayBufferInternal(PyRef<PyArray>);

impl BufferInternal for PyArrayBufferInternal {
fn obj_bytes(&self) -> BorrowedValue<[u8]> {
self.0.get_bytes().into()
}

fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
self.0.get_bytes_mut().into()
}

fn release(&self) {
self.0.exports.fetch_sub(1);
}

fn retain(&self) {
self.0.exports.fetch_add(1);
}
}
static BUFFER_METHODS: BufferMethods = BufferMethods {
obj_bytes: |buffer| buffer.obj_as::<PyArray>().get_bytes().into(),
obj_bytes_mut: |buffer| buffer.obj_as::<PyArray>().get_bytes_mut().into(),
release: |buffer| {
buffer
.obj_as::<PyArray>()
.exports
.fetch_sub(1, atomic::Ordering::Release);
},
retain: |buffer| {
buffer
.obj_as::<PyArray>()
.exports
.fetch_add(1, atomic::Ordering::Release);
},
};

impl AsMapping for PyArray {
fn as_mapping(_zelf: &PyObjectView<Self>, _vm: &VirtualMachine) -> PyMappingMethods {
Expand Down Expand Up @@ -1290,7 +1284,7 @@ mod array {
impl Iterable for PyArray {
fn iter(zelf: PyRef<Self>, vm: &VirtualMachine) -> PyResult {
Ok(PyArrayIter {
position: AtomicCell::new(0),
position: AtomicUsize::new(0),
array: zelf,
}
.into_object(vm))
Expand All @@ -1302,7 +1296,7 @@ mod array {

fn try_resizable(&'a self, vm: &VirtualMachine) -> PyResult<Self::Resizable> {
let w = self.write();
if self.exports.load() == 0 {
if self.exports.load(atomic::Ordering::SeqCst) == 0 {
Ok(w)
} else {
Err(vm.new_buffer_error(
Expand All @@ -1316,7 +1310,7 @@ mod array {
#[pyclass(name = "array_iterator")]
#[derive(Debug, PyValue)]
pub struct PyArrayIter {
position: AtomicCell<usize>,
position: AtomicUsize,
array: PyArrayRef,
}

Expand All @@ -1326,7 +1320,7 @@ mod array {
impl IterNextIterable for PyArrayIter {}
impl IterNext for PyArrayIter {
fn next(zelf: &PyObjectView<Self>, vm: &VirtualMachine) -> PyResult<PyIterReturn> {
let pos = zelf.position.fetch_add(1);
let pos = zelf.position.fetch_add(1, atomic::Ordering::SeqCst);
let r = if let Some(item) = zelf.array.read().getitem_by_idx(pos, vm)? {
PyIterReturn::Return(item)
} else {
Expand Down
89 changes: 47 additions & 42 deletions vm/src/builtins/bytearray.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,6 @@ use super::{
PositionIterInternal, PyBytes, PyBytesRef, PyDictRef, PyIntRef, PyStrRef, PyTuple, PyTupleRef,
PyTypeRef,
};
use crate::common::{
borrow::{BorrowedValue, BorrowedValueMut},
lock::{
PyMappedRwLockReadGuard, PyMappedRwLockWriteGuard, PyMutex, PyRwLock, PyRwLockReadGuard,
PyRwLockWriteGuard,
},
};
use crate::{
anystr::{self, AnyStr},
builtins::PyType,
Expand All @@ -18,9 +11,17 @@ use crate::{
ByteInnerNewOptions, ByteInnerPaddingOptions, ByteInnerSplitOptions,
ByteInnerTranslateOptions, DecodeArgs, PyBytesInner,
},
common::{
atomic::{AtomicUsize, Ordering},
lock::{
PyMappedRwLockReadGuard, PyMappedRwLockWriteGuard, PyMutex, PyRwLock,
PyRwLockReadGuard, PyRwLockWriteGuard,
},
},
function::{ArgBytesLike, ArgIterable, FuncArgs, IntoPyObject, OptionalArg, OptionalOption},
protocol::{
BufferInternal, BufferOptions, BufferResizeGuard, PyBuffer, PyIterReturn, PyMappingMethods,
BufferDescriptor, BufferMethods, BufferResizeGuard, PyBuffer, PyIterReturn,
PyMappingMethods,
},
sliceable::{PySliceableSequence, PySliceableSequenceMut, SequenceIndex},
types::{
Expand All @@ -29,10 +30,9 @@ use crate::{
},
utils::Either,
IdProtocol, PyClassDef, PyClassImpl, PyComparisonValue, PyContext, PyObject, PyObjectRef,
PyRef, PyResult, PyValue, TypeProtocol, VirtualMachine,
PyObjectView, PyObjectWrap, PyRef, PyResult, PyValue, TypeProtocol, VirtualMachine,
};
use bstr::ByteSlice;
use crossbeam_utils::atomic::AtomicCell;
use std::mem::size_of;

/// "bytearray(iterable_of_ints) -> bytearray\n\
Expand All @@ -50,7 +50,7 @@ use std::mem::size_of;
#[derive(Debug, Default)]
pub struct PyByteArray {
inner: PyRwLock<PyBytesInner>,
exports: AtomicCell<usize>,
exports: AtomicUsize,
}

pub type PyByteArrayRef = PyRef<PyByteArray>;
Expand All @@ -63,7 +63,7 @@ impl PyByteArray {
fn from_inner(inner: PyBytesInner) -> Self {
PyByteArray {
inner: PyRwLock::new(inner),
exports: AtomicCell::new(0),
exports: AtomicUsize::new(0),
}
}

Expand Down Expand Up @@ -118,6 +118,12 @@ impl PyByteArray {
Ok(())
}

#[cfg(debug_assertions)]
#[pyproperty]
fn exports(&self) -> usize {
self.exports.load(Ordering::Relaxed)
}

#[inline]
fn inner(&self) -> PyRwLockReadGuard<'_, PyBytesInner> {
self.inner.read()
Expand Down Expand Up @@ -708,36 +714,35 @@ impl Comparable for PyByteArray {
}
}

impl AsBuffer for PyByteArray {
fn as_buffer(zelf: &crate::PyObjectView<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
let buffer = PyBuffer::new(
zelf.as_object().to_owned(),
zelf.to_owned(),
BufferOptions {
readonly: false,
len: zelf.len(),
..Default::default()
},
);
Ok(buffer)
}
}

impl BufferInternal for PyRef<PyByteArray> {
fn obj_bytes(&self) -> BorrowedValue<[u8]> {
self.borrow_buf().into()
}

fn obj_bytes_mut(&self) -> BorrowedValueMut<[u8]> {
PyRwLockWriteGuard::map(self.inner_mut(), |inner| &mut *inner.elements).into()
}

fn release(&self) {
self.exports.fetch_sub(1);
}
static BUFFER_METHODS: BufferMethods = BufferMethods {
obj_bytes: |buffer| buffer.obj_as::<PyByteArray>().borrow_buf().into(),
obj_bytes_mut: |buffer| {
PyMappedRwLockWriteGuard::map(buffer.obj_as::<PyByteArray>().borrow_buf_mut(), |x| {
x.as_mut_slice()
})
.into()
},
release: |buffer| {
buffer
.obj_as::<PyByteArray>()
.exports
.fetch_sub(1, Ordering::Release);
},
retain: |buffer| {
buffer
.obj_as::<PyByteArray>()
.exports
.fetch_add(1, Ordering::Release);
},
};

fn retain(&self) {
self.exports.fetch_add(1);
impl AsBuffer for PyByteArray {
fn as_buffer(zelf: &PyObjectView<Self>, _vm: &VirtualMachine) -> PyResult<PyBuffer> {
Ok(PyBuffer::new(
zelf.to_owned().into_object(),
BufferDescriptor::simple(zelf.len(), false),
&BUFFER_METHODS,
))
}
}

Expand All @@ -746,7 +751,7 @@ impl<'a> BufferResizeGuard<'a> for PyByteArray {

fn try_resizable(&'a self, vm: &VirtualMachine) -> PyResult<Self::Resizable> {
let w = self.inner.upgradable_read();
if self.exports.load() == 0 {
if self.exports.load(Ordering::SeqCst) == 0 {
Ok(parking_lot::lock_api::RwLockUpgradableReadGuard::upgrade(w))
} else {
Err(vm
Expand Down
Loading