Skip to content

Various fixes for datetime arrays. #110

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

Various fixes for datetime arrays. #110

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3c8abbb
Various fixes for datetime arrays.
walshb Jul 12, 2011
a3b9928
Masked arrays now use NaT as default filler value.
walshb Jul 12, 2011
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions numpy/core/_internal.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,11 @@ def _array_descr(descriptor):
else:
new = descriptor.metadata.copy()
# Eliminate any key related to internal implementation
_ = new.pop(METADATA_DTSTR, None)
return (descriptor.str, new)
new.pop(METADATA_DTSTR, None)
if new:
return (descriptor.str, new)
else:
return descriptor.str
else:
return (_array_descr(subdtype[0]), subdtype[1])

Expand Down
33 changes: 24 additions & 9 deletions numpy/lib/_iotools.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -503,10 +503,25 @@ class StringConverter(object):
(_defaulttype, _defaultfunc, _defaultfill) = zip(*_mapper)
#
@classmethod
def _getdtype(cls, val):
"""Returns the dtype of the input variable."""
return np.array(val).dtype
#
@classmethod
def _getsubdtype(cls, val):
"""Returns the type of the dtype of the input variable."""
return np.array(val).dtype.type
#
# This is a bit annoying. We want to return the "general" type in most cases
# (ie. "string" rather than "S10"), but we want to return the specific type
# for datetime64 (ie. "datetime64[us]" rather than "datetime64").
@classmethod
def _dtypeortype(cls, dtype):
"""Returns dtype for datetime64 and type of dtype otherwise."""
if dtype.type == np.datetime64:
return dtype
return dtype.type
#
@classmethod
def upgrade_mapper(cls, func, default=None):
"""
Expand Down Expand Up @@ -561,12 +576,12 @@ def __init__(self, dtype_or_func=None, default=None, missing_values=None,
self.func = str2bool
self._status = 0
self.default = default or False
ttype = np.bool
dtype = np.dtype('bool')
else:
# Is the input a np.dtype ?
try:
self.func = None
ttype = np.dtype(dtype_or_func).type
dtype = np.dtype(dtype_or_func)
except TypeError:
# dtype_or_func must be a function, then
if not hasattr(dtype_or_func, '__call__'):
Expand All @@ -581,11 +596,11 @@ def __init__(self, dtype_or_func=None, default=None, missing_values=None,
default = self.func(asbytes('0'))
except ValueError:
default = None
ttype = self._getsubdtype(default)
dtype = self._getdtype(default)
# Set the status according to the dtype
_status = -1
for (i, (deftype, func, default_def)) in enumerate(self._mapper):
if np.issubdtype(ttype, deftype):
if np.issubdtype(dtype.type, deftype):
_status = i
if default is None:
self.default = default_def
Expand All @@ -603,9 +618,9 @@ def __init__(self, dtype_or_func=None, default=None, missing_values=None,
# If the status is 1 (int), change the function to
# something more robust.
if self.func == self._mapper[1][1]:
if issubclass(ttype, np.uint64):
if issubclass(dtype.type, np.uint64):
self.func = np.uint64
elif issubclass(ttype, np.int64):
elif issubclass(dtype.type, np.int64):
self.func = np.int64
else:
self.func = lambda x : int(float(x))
Expand All @@ -618,7 +633,7 @@ def __init__(self, dtype_or_func=None, default=None, missing_values=None,
self.missing_values = set(list(missing_values) + [asbytes('')])
#
self._callingfunction = self._strict_call
self.type = ttype
self.type = self._dtypeortype(dtype)
self._checked = False
self._initial_default = default
#
Expand Down Expand Up @@ -747,13 +762,13 @@ def update(self, func, default=None, testing_value=None,
# Don't reset the default to None if we can avoid it
if default is not None:
self.default = default
self.type = self._getsubdtype(default)
self.type = self._dtypeortype(self._getdtype(default))
else:
try:
tester = func(testing_value or asbytes('1'))
except (TypeError, ValueError):
tester = None
self.type = self._getsubdtype(tester)
self.type = self._dtypeortype(self._getdtype(tester))
# Add the missing values to the existing set
if missing_values is not None:
if _is_bytes_like(missing_values):
Expand Down
9 changes: 9 additions & 0 deletions numpy/lib/tests/test_io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -776,6 +776,15 @@ def test_converters_cornercases(self):
dtype=[('date', np.object_), ('stid', float)])
assert_equal(test, control)

def test_converters_cornercases2(self):
"Test the conversion to datetime64."
converter = {'date': lambda s: np.datetime64(strptime(s, '%Y-%m-%d %H:%M:%SZ'))}
data = StringIO('2009-02-03 12:00:00Z, 72214.0')
test = np.ndfromtxt(data, delimiter=',', dtype=None,
names=['date', 'stid'], converters=converter)
control = np.array((datetime(2009, 02, 03), 72214.),
dtype=[('date', 'datetime64[us]'), ('stid', float)])
assert_equal(test, control)

def test_unused_converter(self):
"Test whether unused converters are forgotten"
Expand Down
4 changes: 4 additions & 0 deletions numpy/ma/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,8 @@ class MaskError(MAError):
'u' : 999999,
'V' : '???',
'U' : 'N/A',
'M8[D]' : np.datetime64('NaT', 'D'),
'M8[us]' : np.datetime64('NaT', 'us')
}
max_filler = ntypes._minvals
max_filler.update([(k, -np.inf) for k in [np.float32, np.float64]])
Expand Down Expand Up @@ -198,6 +200,8 @@ def default_fill_value(obj):
elif isinstance(obj, np.dtype):
if obj.subdtype:
defval = default_filler.get(obj.subdtype[0].kind, '?')
elif obj.kind == 'M':
defval = default_filler.get(obj.str[1:], '?')
else:
defval = default_filler.get(obj.kind, '?')
elif isinstance(obj, float):
Expand Down