Skip to content

Fix issues with zero-width string fields #6430

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jun 2, 2016
Merged

Fix issues with zero-width string fields #6430

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0bf907a
BUG: Fix numerous bugs related to zero-width string arrays (#473, #19…
embray Oct 8, 2015
7a50050
BUG: Fix indexing a nested structured type that contains a zero-width…
embray May 6, 2016
db7c0b7
BUG: Change ndarray.__new__ to allow zero-width data types, and in pa…
embray May 6, 2016
ee02cdd
BUG: Fixes to reading and writing of empty arrays, and in particular …
embray May 6, 2016
fd95d10
STY: Misc whitespace cleanup.
embray May 6, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions numpy/core/_internal.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,6 @@ def _makenames_list(adict, align):
if (num < 0):
raise ValueError("invalid offset.")
format = dtype(obj[0], align=align)
if (format.itemsize == 0):
raise ValueError("all itemsizes must be fixed.")
if (n > 2):
title = obj[2]
else:
Expand Down
2 changes: 1 addition & 1 deletion numpy/core/records.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -424,7 +424,7 @@ def __new__(subtype, shape, dtype=None, buf=None, offset=0, strides=None,
return self

def __array_finalize__(self, obj):
if self.dtype.type is not record:
if self.dtype.type is not record and self.dtype.fields:
# if self.dtype is not np.record, invoke __setattr__ which will
# convert it to a record if it is a void dtype.
self.dtype = self.dtype
Expand Down
24 changes: 10 additions & 14 deletions numpy/core/src/multiarray/arrayobject.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1652,11 +1652,6 @@ array_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds)
}

itemsize = descr->elsize;
if (itemsize == 0) {
PyErr_SetString(PyExc_ValueError,
"data-type with unspecified variable length");
goto fail;
}

if (strides.ptr != NULL) {
npy_intp nb, off;
Expand Down Expand Up @@ -1690,10 +1685,11 @@ array_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds)

if (buffer.ptr == NULL) {
ret = (PyArrayObject *)
PyArray_NewFromDescr(subtype, descr,
(int)dims.len,
dims.ptr,
strides.ptr, NULL, is_f_order, NULL);
PyArray_NewFromDescr_int(subtype, descr,
(int)dims.len,
dims.ptr,
strides.ptr, NULL, is_f_order, NULL,
0, 1);
if (ret == NULL) {
descr = NULL;
goto fail;
Expand Down Expand Up @@ -1726,11 +1722,11 @@ array_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds)
buffer.flags |= NPY_ARRAY_F_CONTIGUOUS;
}
ret = (PyArrayObject *)\
PyArray_NewFromDescr(subtype, descr,
dims.len, dims.ptr,
strides.ptr,
offset + (char *)buffer.ptr,
buffer.flags, NULL);
PyArray_NewFromDescr_int(subtype, descr,
dims.len, dims.ptr,
strides.ptr,
offset + (char *)buffer.ptr,
buffer.flags, NULL, 0, 1);
if (ret == NULL) {
descr = NULL;
goto fail;
Expand Down
9 changes: 7 additions & 2 deletions numpy/core/src/multiarray/convert.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "npy_pycompat.h"

#include "arrayobject.h"
#include "ctors.h"
#include "mapping.h"
#include "lowlevel_strided_loops.h"
#include "scalartypes.h"
Expand Down Expand Up @@ -132,6 +133,10 @@ PyArray_ToFile(PyArrayObject *self, FILE *fp, char *sep, char *format)
"cannot write object arrays to a file in binary mode");
return -1;
}
if (PyArray_DESCR(self)->elsize == 0) {
/* For zero-width data types there's nothing to write */
return 0;
}
if (npy_fallocate(PyArray_NBYTES(self), fp) != 0) {
return -1;
}
Expand Down Expand Up @@ -600,13 +605,13 @@ PyArray_View(PyArrayObject *self, PyArray_Descr *type, PyTypeObject *pytype)

dtype = PyArray_DESCR(self);
Py_INCREF(dtype);
ret = (PyArrayObject *)PyArray_NewFromDescr(subtype,
ret = (PyArrayObject *)PyArray_NewFromDescr_int(subtype,
dtype,
PyArray_NDIM(self), PyArray_DIMS(self),
PyArray_STRIDES(self),
PyArray_DATA(self),
flags,
(PyObject *)self);
(PyObject *)self, 0, 1);
if (ret == NULL) {
Py_XDECREF(type);
return NULL;
Expand Down
45 changes: 25 additions & 20 deletions numpy/core/src/multiarray/ctors.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -894,10 +894,11 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it,
*
* steals a reference to descr (even on failure)
*/
static PyObject *
NPY_NO_EXPORT PyObject *
PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
npy_intp *dims, npy_intp *strides, void *data,
int flags, PyObject *obj, int zeroed)
int flags, PyObject *obj, int zeroed,
int allow_emptystring)
{
PyArrayObject_fields *fa;
int i;
Expand All @@ -916,7 +917,8 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
newstrides, nd);
ret = PyArray_NewFromDescr_int(subtype, descr, nd, newdims,
newstrides,
data, flags, obj, zeroed);
data, flags, obj, zeroed,
allow_emptystring);
return ret;
}

Expand All @@ -931,20 +933,21 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
/* Check datatype element size */
nbytes = descr->elsize;
if (nbytes == 0) {
if (!PyDataType_ISSTRING(descr)) {
if (!PyDataType_ISFLEXIBLE(descr)) {
PyErr_SetString(PyExc_TypeError, "Empty data-type");
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

from my other comment, I think these lines might be changed to

        if (!PyDataType_ISFLEXIBLE(descr)) {
            PyErr_SetString(PyExc_TypeError, "Empty data-type");
            Py_DECREF(descr);
            return NULL;
        } else if (PyDataType_ISSTRING(descr) && !allow_emptystring) {

Py_DECREF(descr);
return NULL;
}
PyArray_DESCR_REPLACE(descr);
if (descr == NULL) {
return NULL;
}
if (descr->type_num == NPY_STRING) {
nbytes = descr->elsize = 1;
}
else {
nbytes = descr->elsize = sizeof(npy_ucs4);
} else if (PyDataType_ISSTRING(descr) && !allow_emptystring) {
PyArray_DESCR_REPLACE(descr);
if (descr == NULL) {
return NULL;
}
if (descr->type_num == NPY_STRING) {
nbytes = descr->elsize = 1;
}
else {
nbytes = descr->elsize = sizeof(npy_ucs4);
}
}
}

Expand Down Expand Up @@ -1134,7 +1137,7 @@ PyArray_NewFromDescr(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
{
return PyArray_NewFromDescr_int(subtype, descr, nd,
dims, strides, data,
flags, obj, 0);
flags, obj, 0, 0);
}

/*NUMPY_API
Expand Down Expand Up @@ -2855,7 +2858,7 @@ PyArray_Zeros(int nd, npy_intp *dims, PyArray_Descr *type, int is_f_order)
type,
nd, dims,
NULL, NULL,
is_f_order, NULL, 1);
is_f_order, NULL, 1, 0);

if (ret == NULL) {
return NULL;
Expand Down Expand Up @@ -3388,10 +3391,12 @@ PyArray_FromFile(FILE *fp, PyArray_Descr *dtype, npy_intp num, char *sep)
return NULL;
}
if (dtype->elsize == 0) {
PyErr_SetString(PyExc_ValueError,
"The elements are 0-sized.");
Py_DECREF(dtype);
return NULL;
/* Nothing to read, just create an empty array of the requested type */
return PyArray_NewFromDescr_int(&PyArray_Type,
dtype,
1, &num,
NULL, NULL,
0, NULL, 0, 1);
}
if ((sep == NULL) || (strlen(sep) == 0)) {
ret = array_fromfile_binary(fp, dtype, num, &nread);
Expand Down
6 changes: 6 additions & 0 deletions numpy/core/src/multiarray/ctors.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,12 @@ PyArray_NewFromDescr(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
npy_intp *dims, npy_intp *strides, void *data,
int flags, PyObject *obj);

NPY_NO_EXPORT PyObject *
PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
npy_intp *dims, npy_intp *strides, void *data,
int flags, PyObject *obj, int zeroed,
int allow_emptystring);

NPY_NO_EXPORT PyObject *PyArray_New(PyTypeObject *, int nd, npy_intp *,
int, npy_intp *, void *, int, int, PyObject *);

Expand Down
2 changes: 1 addition & 1 deletion numpy/core/src/multiarray/descriptor.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1138,7 +1138,7 @@ _convert_from_dict(PyObject *obj, int align)
}
}
Py_DECREF(tup);
if ((ret == NPY_FAIL) || (newdescr->elsize == 0)) {
if (ret == NPY_FAIL) {
goto fail;
}
dtypeflags |= (newdescr->flags & NPY_FROM_FIELDS);
Expand Down
13 changes: 7 additions & 6 deletions numpy/core/src/multiarray/dtype_transfer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include "npy_pycompat.h"

#include "convert_datatype.h"
#include "ctors.h"
#include "_datetime.h"
#include "datetime_strings.h"

Expand Down Expand Up @@ -549,8 +550,8 @@ wrap_copy_swap_function(int aligned,
* The copyswap functions shouldn't need that.
*/
Py_INCREF(dtype);
data->arr = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype,
1, &shape, NULL, NULL, 0, NULL);
data->arr = (PyArrayObject *)PyArray_NewFromDescr_int(&PyArray_Type, dtype,
1, &shape, NULL, NULL, 0, NULL, 0, 1);
if (data->arr == NULL) {
PyArray_free(data);
return NPY_FAIL;
Expand Down Expand Up @@ -1405,8 +1406,8 @@ get_nbo_cast_transfer_function(int aligned,
return NPY_FAIL;
}
}
data->aip = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, tmp_dtype,
1, &shape, NULL, NULL, 0, NULL);
data->aip = (PyArrayObject *)PyArray_NewFromDescr_int(&PyArray_Type,
tmp_dtype, 1, &shape, NULL, NULL, 0, NULL, 0, 1);
if (data->aip == NULL) {
PyArray_free(data);
return NPY_FAIL;
Expand All @@ -1429,8 +1430,8 @@ get_nbo_cast_transfer_function(int aligned,
return NPY_FAIL;
}
}
data->aop = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, tmp_dtype,
1, &shape, NULL, NULL, 0, NULL);
data->aop = (PyArrayObject *)PyArray_NewFromDescr_int(&PyArray_Type,
tmp_dtype, 1, &shape, NULL, NULL, 0, NULL, 0, 1);
if (data->aop == NULL) {
Py_DECREF(data->aip);
PyArray_free(data);
Expand Down
9 changes: 5 additions & 4 deletions numpy/core/src/multiarray/mapping.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "npy_import.h"

#include "common.h"
#include "ctors.h"
#include "iterators.h"
#include "mapping.h"
#include "lowlevel_strided_loops.h"
Expand Down Expand Up @@ -1291,15 +1292,15 @@ _get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view)

/* view the array at the new offset+dtype */
Py_INCREF(fieldtype);
*view = (PyArrayObject*)PyArray_NewFromDescr(
*view = (PyArrayObject*)PyArray_NewFromDescr_int(
Py_TYPE(arr),
fieldtype,
PyArray_NDIM(arr),
PyArray_SHAPE(arr),
PyArray_STRIDES(arr),
PyArray_BYTES(arr) + offset,
PyArray_FLAGS(arr),
(PyObject *)arr);
(PyObject *)arr, 0, 1);
if (*view == NULL) {
return 0;
}
Expand Down Expand Up @@ -1397,15 +1398,15 @@ _get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view)
view_dtype->fields = fields;
view_dtype->flags = PyArray_DESCR(arr)->flags;

*view = (PyArrayObject*)PyArray_NewFromDescr(
*view = (PyArrayObject*)PyArray_NewFromDescr_int(
Py_TYPE(arr),
view_dtype,
PyArray_NDIM(arr),
PyArray_SHAPE(arr),
PyArray_STRIDES(arr),
PyArray_DATA(arr),
PyArray_FLAGS(arr),
(PyObject *)arr);
(PyObject *)arr, 0, 1);
if (*view == NULL) {
return 0;
}
Expand Down
14 changes: 7 additions & 7 deletions numpy/core/src/multiarray/methods.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -379,13 +379,13 @@ PyArray_GetField(PyArrayObject *self, PyArray_Descr *typed, int offset)
Py_DECREF(safe);
}

ret = PyArray_NewFromDescr(Py_TYPE(self),
typed,
PyArray_NDIM(self), PyArray_DIMS(self),
PyArray_STRIDES(self),
PyArray_BYTES(self) + offset,
PyArray_FLAGS(self)&(~NPY_ARRAY_F_CONTIGUOUS),
(PyObject *)self);
ret = PyArray_NewFromDescr_int(Py_TYPE(self),
typed,
PyArray_NDIM(self), PyArray_DIMS(self),
PyArray_STRIDES(self),
PyArray_BYTES(self) + offset,
PyArray_FLAGS(self)&(~NPY_ARRAY_F_CONTIGUOUS),
(PyObject *)self, 0, 1);
if (ret == NULL) {
return NULL;
}
Expand Down
4 changes: 2 additions & 2 deletions numpy/core/src/multiarray/shape.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,12 +255,12 @@ PyArray_Newshape(PyArrayObject *self, PyArray_Dims *newdims,
}

Py_INCREF(PyArray_DESCR(self));
ret = (PyArrayObject *)PyArray_NewFromDescr(Py_TYPE(self),
ret = (PyArrayObject *)PyArray_NewFromDescr_int(Py_TYPE(self),
PyArray_DESCR(self),
ndim, dimensions,
strides,
PyArray_DATA(self),
flags, (PyObject *)self);
flags, (PyObject *)self, 0, 1);

if (ret == NULL) {
goto fail;
Expand Down
10 changes: 10 additions & 0 deletions numpy/core/tests/test_dtype.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,16 @@ def test_from_dictproxy(self):
dt2 = np.dtype((np.void, dt.fields))
assert_equal(dt2.fields, dt.fields)

def test_from_dict_with_zero_width_field(self):
# Regression test for #6430 / #2196
dt = np.dtype([('val1', np.float32, (0,)), ('val2', int)])
dt2 = np.dtype({'names': ['val1', 'val2'],
'formats': [(np.float32, (0,)), int]})

assert_dtype_equal(dt, dt2)
assert_equal(dt.fields['val1'][0].itemsize, 0)
assert_equal(dt.itemsize, dt.fields['val2'][0].itemsize)

def test_bool_commastring(self):
d = np.dtype('?,?,?') # raises?
assert_equal(len(d.names), 3)
Expand Down
Loading