Skip to content

BUG: Do not accidentally store dtype metadata in np.save #27151

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 9, 2024
Merged

BUG: Do not accidentally store dtype metadata in np.save #27151

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions numpy/lib/format.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,8 @@ def dtype_to_descr(dtype):
warnings.warn("metadata on a dtype is not saved to an npy/npz. "
"Use another format (such as pickle) to store it.",
UserWarning, stacklevel=2)
dtype = new_dtype

if dtype.names is not None:
# This is a record array. The .descr is fine. XXX: parts of the
# record array with an empty name, like padding bytes, still get
Expand Down
34 changes: 16 additions & 18 deletions numpy/lib/tests/test_format.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -998,32 +998,30 @@ def test_header_growth_axis():

assert len(fp.getvalue()) == expected_header_length

@pytest.mark.parametrize('dt, fail', [
(np.dtype({'names': ['a', 'b'], 'formats': [float, np.dtype('S3',
metadata={'some': 'stuff'})]}), True),
(np.dtype(int, metadata={'some': 'stuff'}), False),
(np.dtype([('subarray', (int, (2,)))], metadata={'some': 'stuff'}), False),
@pytest.mark.parametrize('dt', [
np.dtype({'names': ['a', 'b'], 'formats': [float, np.dtype('S3',
metadata={'some': 'stuff'})]}),
np.dtype(int, metadata={'some': 'stuff'}),
np.dtype([('subarray', (int, (2,)))], metadata={'some': 'stuff'}),
# recursive: metadata on the field of a dtype
(np.dtype({'names': ['a', 'b'], 'formats': [
np.dtype({'names': ['a', 'b'], 'formats': [
float, np.dtype({'names': ['c'], 'formats': [np.dtype(int, metadata={})]})
]}), False)
]}),
])
@pytest.mark.skipif(IS_PYPY and sys.implementation.version <= (7, 3, 8),
reason="PyPy bug in error formatting")
def test_metadata_dtype(dt, fail):
def test_metadata_dtype(dt):
# gh-14142
arr = np.ones(10, dtype=dt)
buf = BytesIO()
with assert_warns(UserWarning):
np.save(buf, arr)
buf.seek(0)
if fail:
with assert_raises(ValueError):
np.load(buf)
else:
arr2 = np.load(buf)
# BUG: assert_array_equal does not check metadata
from numpy.lib._utils_impl import drop_metadata
assert_array_equal(arr, arr2)
assert drop_metadata(arr.dtype) is not arr.dtype
assert drop_metadata(arr2.dtype) is arr2.dtype

# Loading should work (metadata was stripped):
arr2 = np.load(buf)
# BUG: assert_array_equal does not check metadata
from numpy.lib._utils_impl import drop_metadata
assert_array_equal(arr, arr2)
assert drop_metadata(arr.dtype) is not arr.dtype
assert drop_metadata(arr2.dtype) is arr2.dtype
2 changes: 1 addition & 1 deletion numpy/lib/tests/test_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def _compare_dtypes(dt1, dt2):
assert dt_m.metadata is None
assert dt_m['l1'].metadata is None
assert dt_m['l1']['l2'].metadata is None

# alignment
dt = np.dtype([('x', '<f8'), ('y', '<i4')],
align=True,
Expand Down
Loading