Skip to content

BUG: Fix NEP50 weak-type scalar logic for binary operations on masked arrays #28567

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
+215 −19
Open

BUG: Fix NEP50 weak-type scalar logic for binary operations on masked arrays #28567

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
74cb500
Add NEP50 logic for masked arrays and tests
rhugonnet Mar 21, 2025
738bb00
Remove space edits from editor in test file
rhugonnet Mar 21, 2025
3ab5f4d
Further fix edited spaces
rhugonnet Mar 21, 2025
f39ffc7
Final fix for spaces
rhugonnet Mar 21, 2025
901fe80
Use np.ufunc.resolve_dtypes instead of placeholder operation
rhugonnet Apr 4, 2025
a77d5c3
Reverse trailing space removal from automatic editor linter
rhugonnet Apr 4, 2025
1748a80
Merge branch 'main' into bug_nep50_ma
rhugonnet Aug 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 55 additions & 19 deletions numpy/ma/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -718,7 +718,7 @@ def get_masked_subclass(*arrays):
return rcls


def getdata(a, subok=True):
def getdata(a, subok=True, return_pyscalar=False):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seeing as it's being used as keyword-only, so maybe we should just enforce it?

Suggested change
def getdata(a, subok=True, return_pyscalar=False):
def getdata(a, subok=True, *, return_pyscalar=False):

"""
Return the data of a masked array as an ndarray.

Expand All @@ -732,6 +732,9 @@ def getdata(a, subok=True):
subok : bool
Whether to force the output to be a `pure` ndarray (False) or to
return a subclass of ndarray if appropriate (True, default).
return_pyscalar : bool
Whether to force the ouput to remain a Python scalar if the input was
also a Python scalar (True), or to return an array in any case (False, default).

See Also
--------
Expand Down Expand Up @@ -764,7 +767,10 @@ def getdata(a, subok=True):
try:
data = a._data
except AttributeError:
data = np.array(a, copy=None, subok=subok)
if type(a) in (int, float, complex) and return_pyscalar:
data = a
else:
data = np.array(a, copy=None, subok=subok)
if not subok:
return data.view(ndarray)
return data
Expand Down Expand Up @@ -1066,7 +1072,7 @@ def __call__(self, a, b, *args, **kwargs):

"""
# Get the data, as ndarray
(da, db) = (getdata(a), getdata(b))
(da, db) = (getdata(a, return_pyscalar=True), getdata(b, return_pyscalar=True))
# Get the result
with np.errstate():
np.seterr(divide='ignore', invalid='ignore')
Expand Down Expand Up @@ -1142,7 +1148,7 @@ def outer(self, a, b):
Return the function applied to the outer product of a and b.

"""
(da, db) = (getdata(a), getdata(b))
(da, db) = (getdata(a, return_pyscalar=True), getdata(b, return_pyscalar=True))
d = self.f.outer(da, db)
ma = getmask(a)
mb = getmask(b)
Expand Down Expand Up @@ -1209,7 +1215,7 @@ def __init__(self, dbfunc, domain, fillx=0, filly=0):
def __call__(self, a, b, *args, **kwargs):
"Execute the call behavior."
# Get the data
(da, db) = (getdata(a), getdata(b))
(da, db) = (getdata(a, return_pyscalar=True), getdata(b, return_pyscalar=True))
# Get the result
with np.errstate(divide='ignore', invalid='ignore'):
result = self.f(da, db, *args, **kwargs)
Expand Down Expand Up @@ -4424,8 +4430,13 @@ def __iadd__(self, other):
self._mask += m
elif m is not nomask:
self._mask += m
other_data = getdata(other)
other_data = np.where(self._mask, other_data.dtype.type(0), other_data)
other_data = getdata(other, return_pyscalar=True)
if type(other_data) in (int, float, complex):
other_dtype = np.add.resolve_dtypes(
(self._data.dtype, type(other_data), None), casting="unsafe")[1]
else:
other_dtype = other_data.dtype
other_data = np.where(self._mask, other_dtype.type(0), other_data)
self._data.__iadd__(other_data)
return self

Expand All @@ -4441,8 +4452,13 @@ def __isub__(self, other):
self._mask += m
elif m is not nomask:
self._mask += m
other_data = getdata(other)
other_data = np.where(self._mask, other_data.dtype.type(0), other_data)
other_data = getdata(other, return_pyscalar=True)
if type(other_data) in (int, float, complex):
other_dtype = np.subtract.resolve_dtypes(
(self._data.dtype, type(other_data), None), casting="unsafe")[1]
else:
other_dtype = other_data.dtype
other_data = np.where(self._mask, other_dtype.type(0), other_data)
self._data.__isub__(other_data)
return self

Expand All @@ -4458,8 +4474,13 @@ def __imul__(self, other):
self._mask += m
elif m is not nomask:
self._mask += m
other_data = getdata(other)
other_data = np.where(self._mask, other_data.dtype.type(1), other_data)
other_data = getdata(other, return_pyscalar=True)
if type(other_data) in (int, float, complex):
other_dtype = np.multiply.resolve_dtypes(
(self._data.dtype, type(other_data), None), casting="unsafe")[1]
else:
other_dtype = other_data.dtype
other_data = np.where(self._mask, other_dtype.type(1), other_data)
self._data.__imul__(other_data)
return self

Expand All @@ -4468,17 +4489,22 @@ def __ifloordiv__(self, other):
Floor divide self by other in-place.

"""
other_data = getdata(other)
other_data = getdata(other, return_pyscalar=True)
if type(other_data) in (int, float, complex):
other_dtype = np.floor_divide.resolve_dtypes(
(self._data.dtype, type(other_data), None), casting="unsafe")[1]
else:
other_dtype = other_data.dtype
dom_mask = _DomainSafeDivide().__call__(self._data, other_data)
other_mask = getmask(other)
new_mask = mask_or(other_mask, dom_mask)
# The following 3 lines control the domain filling
if dom_mask.any():
(_, fval) = ufunc_fills[np.floor_divide]
other_data = np.where(
dom_mask, other_data.dtype.type(fval), other_data)
dom_mask, other_dtype.type(fval), other_data)
self._mask |= new_mask
other_data = np.where(self._mask, other_data.dtype.type(1), other_data)
other_data = np.where(self._mask, other_dtype.type(1), other_data)
self._data.__ifloordiv__(other_data)
return self

Expand All @@ -4487,17 +4513,22 @@ def __itruediv__(self, other):
True divide self by other in-place.

"""
other_data = getdata(other)
other_data = getdata(other, return_pyscalar=True)
if type(other_data) in (int, float, complex):
other_dtype = np.true_divide.resolve_dtypes(
(self._data.dtype, type(other_data), None), casting="unsafe")[1]
else:
other_dtype = other_data.dtype
dom_mask = _DomainSafeDivide().__call__(self._data, other_data)
other_mask = getmask(other)
new_mask = mask_or(other_mask, dom_mask)
# The following 3 lines control the domain filling
if dom_mask.any():
(_, fval) = ufunc_fills[np.true_divide]
other_data = np.where(
dom_mask, other_data.dtype.type(fval), other_data)
dom_mask, other_dtype.type(fval), other_data)
self._mask |= new_mask
other_data = np.where(self._mask, other_data.dtype.type(1), other_data)
other_data = np.where(self._mask, other_dtype.type(1), other_data)
self._data.__itruediv__(other_data)
return self

Expand All @@ -4506,8 +4537,13 @@ def __ipow__(self, other):
Raise self to the power other, in place.

"""
other_data = getdata(other)
other_data = np.where(self._mask, other_data.dtype.type(1), other_data)
other_data = getdata(other, return_pyscalar=True)
if type(other_data) in (int, float, complex):
other_dtype = np.power.resolve_dtypes(
(self._data.dtype, type(other_data), None), casting="unsafe")[1]
else:
other_dtype = other_data.dtype
other_data = np.where(self._mask, other_dtype.type(1), other_data)
other_mask = getmask(other)
with np.errstate(divide='ignore', invalid='ignore'):
self._data.__ipow__(other_data)
Expand Down
160 changes: 160 additions & 0 deletions numpy/ma/tests/test_core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5949,3 +5949,163 @@ def test_uint_fill_value_and_filled():
# And this ensures things like filled work:
np.testing.assert_array_equal(
a.filled(), np.array([999999, 1]).astype("uint16"), strict=True)


@pytest.mark.skipif(IS_WASM, reason="wasm doesn't have support for fp errors")
def test_nep50_examples():
"""Adapted for masked arrays from numpy ndarray NP50 tests."""

res = np.ma.masked_array([1], dtype=np.uint8) + 2
assert res.dtype == np.uint8

res = np.ma.masked_array([1], dtype=np.uint8) + np.int64(1)
assert res.dtype == np.int64

res = np.ma.masked_array([1], dtype=np.uint8) + np.ma.masked_array(1, dtype=np.int64)
assert res.dtype == np.int64

res = np.ma.masked_array([0.1], dtype=np.float32) == np.float64(0.1)
assert res[0] == False

res = np.ma.masked_array([0.1], dtype=np.float32) + np.float64(0.1)
assert res.dtype == np.float64

res = np.ma.masked_array([1.], dtype=np.float32) + np.int64(3)
assert res.dtype == np.float64


@pytest.mark.parametrize("dtype", np.typecodes["AllInteger"])
def test_nep50_weak_integers(dtype):
"""Adapted for masked arrays from numpy ndarray NP50 tests."""

maxint = int(np.iinfo(dtype).max)

# Array operations are not expected to warn, but should give the same
# result dtype.
res = np.ma.masked_array([100], dtype=dtype) + maxint
assert res.dtype == dtype


@pytest.mark.parametrize("dtype", np.typecodes["AllFloat"])
def test_nep50_weak_integers_with_inexact(dtype):
"""Adapted for masked arrays from numpy ndarray NP50 tests."""

too_big_int = int(np.finfo(dtype).max) * 2

if dtype in "dDG":
# These dtypes currently convert to Python float internally, which
# raises an OverflowError, while the other dtypes overflow to inf.
# NOTE: It may make sense to normalize the behavior!
with pytest.raises(OverflowError):
np.ma.masked_array([1], dtype=dtype) + too_big_int
else:
# NumPy uses (or used) `int -> string -> longdouble` for the
# conversion. But Python may refuse `str(int)` for huge ints.
# In that case, RuntimeWarning would be correct, but conversion
# fails earlier (seems to happen on 32bit linux, possibly only debug).
if dtype in "gG":
try:
str(too_big_int)
except ValueError:
pytest.skip("`huge_int -> string -> longdouble` failed")

with pytest.warns(RuntimeWarning):
# We force the dtype here, since windows may otherwise pick the
# double instead of the longdouble loop. That leads to slightly
# different results (conversion of the int fails as above).
res = np.add(np.ma.masked_array([1], dtype=dtype), too_big_int, dtype=dtype)
assert res.dtype == dtype
assert res == np.inf


def test_nep50_integer_conversion_errors():
"""Adapted for masked arrays from numpy ndarray NP50 tests."""

# Implementation for error paths is mostly missing (as of writing)
with pytest.raises(OverflowError, match=".*uint8"):
np.ma.masked_array([1], dtype=np.uint8) + 300


def test_nep50_with_axisconcatenator():
"""Adapted for masked arrays from numpy ndarray NP50 tests."""

# Concatenate/r_ does not promote, so this has to error:
with pytest.raises(OverflowError):
np.r_[np.ma.arange(5, dtype=np.int8), 255]


@pytest.mark.parametrize("arr", [
np.ma.ones((100, 100), dtype=np.uint8)[::2], # not trivially iterable
np.ma.ones(20000, dtype=">u4"), # cast and >buffersize
np.ma.ones(100, dtype=">u4"), # fast path compatible with cast
])
def test_integer_comparison_with_cast(arr):
"""Adapted for masked arrays from numpy ndarray NP50 tests."""

# Similar to above, but mainly test a few cases that cover the slow path
# the test is limited to unsigned ints and -1 for simplicity.
res = arr >= -1
assert_array_equal(res.data, np.ones_like(arr, dtype=bool))
res = arr < -1
assert_array_equal(res.data, np.zeros_like(arr, dtype=bool))


def create_with_ma_array(sctype, value):
return np.ma.masked_array([value], dtype=sctype)

@pytest.mark.parametrize("sctype",
[np.int8, np.int16, np.int32, np.int64,
np.uint8, np.uint16, np.uint32, np.uint64])
@pytest.mark.parametrize("create", [create_with_ma_array])
def test_oob_creation(sctype, create):
"""Adapted for masked arrays from numpy ndarray NP50 tests."""

iinfo = np.iinfo(sctype)

with pytest.raises(OverflowError):
create(sctype, iinfo.min - 1)

with pytest.raises(OverflowError):
create(sctype, iinfo.max + 1)

with pytest.raises(OverflowError):
create(sctype, str(iinfo.min - 1))

with pytest.raises(OverflowError):
create(sctype, str(iinfo.max + 1))

assert create(sctype, iinfo.min) == iinfo.min
assert create(sctype, iinfo.max) == iinfo.max

@pytest.mark.parametrize("operator", ["__iadd__", "__isub__", "__imul__",
"__ifloordiv__", "__itruediv__", "__ipow__"])
@pytest.mark.parametrize("scalar", [1, 1.])
@pytest.mark.parametrize("dtype", [np.uint8, np.int8, np.uint32, np.int32, np.float32, np.float64])
def test_nep50_inplace_ma(operator, scalar, dtype):
"""Check that in-place operations respect NEP50 like ndarrays do."""

# Define ndarray, and apply in-place operation with scalar, capturing exceptions
arr = np.array([1], dtype=dtype)
try:
getattr(arr, operator)(scalar)
msg = None
except (OverflowError, np._core._exceptions._UFuncOutputCastingError) as e:
# Skip case where casting error occurs
if isinstance(e, np._core._exceptions._UFuncOutputCastingError):
return
# Otherwise continue to check overflow error of NEP50
else:
msg = str(e)

# Define masked array, and apply the same operation with the same scalar
ma_arr = np.ma.masked_array([1], dtype=dtype)
# If overflow error should be raised,
if msg is not None:
with pytest.raises(OverflowError, match=msg):
getattr(ma_arr, operator)(scalar)
# Otherwise, apply in-place operation
else:
getattr(ma_arr, operator)(scalar)

# Output data types should be the same
assert ma_arr.dtype == arr.dtype
Loading