From e64b6f1c8275f06c94c0b83f7b5469d7a33cd639 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Tue, 7 Jun 2011 15:22:38 -0500 Subject: [PATCH 01/34] ENH: datetime-autounit: Automatically detect the unit for scalar construction --- numpy/core/src/multiarray/_datetime.h | 44 ++- numpy/core/src/multiarray/datetime.c | 323 +++++++++++++++++--- numpy/core/src/multiarray/scalartypes.c.src | 52 +--- numpy/core/tests/test_datetime.py | 87 +++++- 4 files changed, 423 insertions(+), 83 deletions(-) diff --git a/numpy/core/src/multiarray/_datetime.h b/numpy/core/src/multiarray/_datetime.h index 519a1c545aa5..b07fe2f514f6 100644 --- a/numpy/core/src/multiarray/_datetime.h +++ b/numpy/core/src/multiarray/_datetime.h @@ -175,23 +175,46 @@ append_metastr_to_string(PyArray_DatetimeMetaData *meta, NPY_NO_EXPORT int get_datetime_iso_8601_strlen(int local, NPY_DATETIMEUNIT base); - /* * Parses (almost) standard ISO 8601 date strings. The differences are: * + * + After the date and time, may place a ' ' followed by an event number. * + The date "20100312" is parsed as the year 20100312, not as * equivalent to "2010-03-12". The '-' in the dates are not optional. * + Only seconds may have a decimal point, with up to 18 digits after it * (maximum attoseconds precision). * + Either a 'T' as in ISO 8601 or a ' ' may be used to separate * the date and the time. Both are treated equivalently. + * + Doesn't (yet) handle the "YYYY-DDD" or "YYYY-Www" formats. + * + Doesn't handle leap seconds (seconds value has 60 in these cases). + * + Doesn't handle 24:00:00 as synonym for midnight (00:00:00) tomorrow + * + Accepts special values "NaT" (not a time), "Today", (current + * day according to local time) and "Now" (current time in UTC). * * 'str' must be a NULL-terminated string, and 'len' must be its length. * + * 'out' gets filled with the parsed date-time. + * 'out_local' gets set to 1 if the parsed time was in local time, + * to 0 otherwise. The values 'now' and 'today' don't get counted + * as local, and neither do UTC +/-#### timezone offsets, because + * they aren't using the computer's local timezone offset. + * 'out_bestunit' gives a suggested unit based on the amount of + * resolution provided in the string, or -1 for NaT. + * 'out_special' gets set to 1 if the parsed time was 'today', + * 'now', or ''/'NaT'. For 'today', the unit recommended is + * 'D', for 'now', the unit recommended is 's', and for 'NaT' + * the unit recommended is 'Y'. + * + * * Returns 0 on success, -1 on failure. */ NPY_NO_EXPORT int -parse_iso_8601_date(char *str, int len, npy_datetimestruct *out); +parse_iso_8601_date(char *str, int len, + npy_datetimestruct *out, + npy_bool *out_local, + NPY_DATETIMEUNIT *out_bestunit, + npy_bool *out_special); + /* * Converts an npy_datetimestruct to an (almost) ISO 8601 @@ -214,19 +237,26 @@ NPY_NO_EXPORT int make_iso_8601_date(npy_datetimestruct *dts, char *outstr, int outlen, int local, NPY_DATETIMEUNIT base, int tzoffset); - /* * Tests for and converts a Python datetime.datetime or datetime.date * object into a NumPy npy_datetimestruct. * + * 'out_bestunit' gives a suggested unit based on whether the object + * was a datetime.date or datetime.datetime object. + * * Returns -1 on error, 0 on success, and 1 (with no error set) * if obj doesn't have the neeeded date or datetime attributes. */ NPY_NO_EXPORT int -convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out); +convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, + NPY_DATETIMEUNIT *out_bestunit); /* - * Converts a PyObject * into a datetime, in any of the input forms supported. + * Converts a PyObject * into a datetime, in any of the forms supported. + * + * If the units metadata isn't known ahead of time, set meta->base + * to -1, and this function will populate meta with either default + * values or values from the input object. * * Returns -1 on error, 0 on success. */ @@ -237,6 +267,10 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, /* * Converts a PyObject * into a timedelta, in any of the forms supported * + * If the units metadata isn't known ahead of time, set meta->base + * to -1, and this function will populate meta with either default + * values or values from the input object. + * * Returns -1 on error, 0 on success. */ NPY_NO_EXPORT int diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index a1fcd54fad66..8e57961eb85c 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -1168,8 +1168,8 @@ parse_datetime_metacobj_from_metastr(char *metastr, Py_ssize_t len) /* If there's no metastr, use the default */ if (len == 0) { - dt_data->num = 1; dt_data->base = NPY_DATETIME_DEFAULTUNIT; + dt_data->num = 1; dt_data->events = 1; } else { @@ -2279,13 +2279,30 @@ add_minutes_to_datetimestruct(npy_datetimestruct *dts, int minutes) * * 'str' must be a NULL-terminated string, and 'len' must be its length. * + * 'out' gets filled with the parsed date-time. + * 'out_local' gets set to 1 if the parsed time was in local time, + * to 0 otherwise. The values 'now' and 'today' don't get counted + * as local, and neither do UTC +/-#### timezone offsets, because + * they aren't using the computer's local timezone offset. + * 'out_bestunit' gives a suggested unit based on the amount of + * resolution provided in the string, or -1 for NaT. + * 'out_special' gets set to 1 if the parsed time was 'today', + * 'now', or ''/'NaT'. For 'today', the unit recommended is + * 'D', for 'now', the unit recommended is 's', and for 'NaT' + * the unit recommended is 'Y'. + * + * * Returns 0 on success, -1 on failure. */ NPY_NO_EXPORT int -parse_iso_8601_date(char *str, int len, npy_datetimestruct *out) +parse_iso_8601_date(char *str, int len, + npy_datetimestruct *out, + npy_bool *out_local, + NPY_DATETIMEUNIT *out_bestunit, + npy_bool *out_special) { int year_leap = 0; - int i; + int i, numdigits; char *substr, sublen; /* Initialize the output to all zeros */ @@ -2299,6 +2316,22 @@ parse_iso_8601_date(char *str, int len, npy_datetimestruct *out) tolower(str[1]) == 'a' && tolower(str[2]) == 't')) { out->year = NPY_DATETIME_NAT; + + /* + * Indicate that this was a special value, and + * recommend 'Y' as the unit because when promoted + * with any other unit, will produce that unit. + */ + if (out_local != NULL) { + *out_local = 0; + } + if (out_bestunit != NULL) { + *out_bestunit = NPY_FR_Y; + } + if (out_special != NULL) { + *out_special = 1; + } + return 0; } @@ -2335,6 +2368,21 @@ parse_iso_8601_date(char *str, int len, npy_datetimestruct *out) out->year = tm_.tm_year + 1900; out->month = tm_.tm_mon + 1; out->day = tm_.tm_mday; + + /* + * Indicate that this was a special value, and + * is a date (unit 'D'). + */ + if (out_local != NULL) { + *out_local = 0; + } + if (out_bestunit != NULL) { + *out_bestunit = NPY_FR_D; + } + if (out_special != NULL) { + *out_special = 1; + } + return 0; } @@ -2351,9 +2399,29 @@ parse_iso_8601_date(char *str, int len, npy_datetimestruct *out) meta.num = 1; meta.events = 1; + /* + * Indicate that this was a special value, and + * use 's' because the time() function has resolution + * seconds. + */ + if (out_local != NULL) { + *out_local = 0; + } + if (out_bestunit != NULL) { + *out_bestunit = NPY_FR_s; + } + if (out_special != NULL) { + *out_special = 1; + } + return convert_datetime_to_datetimestruct(&meta, rawtime, out); } + /* Anything else isn't a special value */ + if (out_special != NULL) { + *out_special = 0; + } + substr = str; sublen = len; @@ -2390,6 +2458,12 @@ parse_iso_8601_date(char *str, int len, npy_datetimestruct *out) /* Next character must be a '-' or the end of the string */ if (sublen == 0) { + if (out_local != NULL) { + *out_local = 0; + } + if (out_bestunit != NULL) { + *out_bestunit = NPY_FR_Y; + } goto finish; } else if (*substr == '-') { @@ -2423,6 +2497,12 @@ parse_iso_8601_date(char *str, int len, npy_datetimestruct *out) /* Next character must be a '-' or the end of the string */ if (sublen == 0) { + if (out_local != NULL) { + *out_local = 0; + } + if (out_bestunit != NULL) { + *out_bestunit = NPY_FR_M; + } goto finish; } else if (*substr == '-') { @@ -2457,6 +2537,12 @@ parse_iso_8601_date(char *str, int len, npy_datetimestruct *out) /* Next character must be a 'T', ' ', or end of string */ if (sublen == 0) { + if (out_local != NULL) { + *out_local = 0; + } + if (out_bestunit != NULL) { + *out_bestunit = NPY_FR_D; + } goto finish; } else if (*substr != 'T' && *substr != ' ') { @@ -2489,6 +2575,9 @@ parse_iso_8601_date(char *str, int len, npy_datetimestruct *out) --sublen; } else { + if (out_bestunit != NULL) { + *out_bestunit = NPY_FR_h; + } goto parse_timezone; } @@ -2519,6 +2608,9 @@ parse_iso_8601_date(char *str, int len, npy_datetimestruct *out) --sublen; } else { + if (out_bestunit != NULL) { + *out_bestunit = NPY_FR_m; + } goto parse_timezone; } @@ -2549,44 +2641,78 @@ parse_iso_8601_date(char *str, int len, npy_datetimestruct *out) --sublen; } else { + if (out_bestunit != NULL) { + *out_bestunit = NPY_FR_s; + } goto parse_timezone; } /* PARSE THE MICROSECONDS (0 to 6 digits) */ + numdigits = 0; for (i = 0; i < 6; ++i) { out->us *= 10; if (sublen > 0 && isdigit(*substr)) { out->us += (*substr - '0'); ++substr; --sublen; + ++numdigits; } } if (sublen == 0 || !isdigit(*substr)) { + if (out_bestunit != NULL) { + if (numdigits > 3) { + *out_bestunit = NPY_FR_us; + } + else { + *out_bestunit = NPY_FR_ms; + } + } goto parse_timezone; } /* PARSE THE PICOSECONDS (0 to 6 digits) */ + numdigits = 0; for (i = 0; i < 6; ++i) { out->ps *= 10; if (sublen > 0 && isdigit(*substr)) { out->ps += (*substr - '0'); ++substr; --sublen; + ++numdigits; } } if (sublen == 0 || !isdigit(*substr)) { + if (out_bestunit != NULL) { + if (numdigits > 3) { + *out_bestunit = NPY_FR_ps; + } + else { + *out_bestunit = NPY_FR_ns; + } + } goto parse_timezone; } /* PARSE THE ATTOSECONDS (0 to 6 digits) */ + numdigits = 0; for (i = 0; i < 6; ++i) { out->as *= 10; if (sublen > 0 && isdigit(*substr)) { out->as += (*substr - '0'); ++substr; --sublen; + ++numdigits; + } + } + + if (out_bestunit != NULL) { + if (numdigits > 3) { + *out_bestunit = NPY_FR_as; + } + else { + *out_bestunit = NPY_FR_fs; } } @@ -2642,11 +2768,21 @@ parse_iso_8601_date(char *str, int len, npy_datetimestruct *out) out->year = tm_.tm_year + 1900; } + /* Since neither "Z" nor a time-zone was specified, it's local */ + if (out_local != NULL) { + *out_local = 1; + } + goto finish; } /* UTC specifier */ if (*substr == 'Z') { + /* "Z" means not local */ + if (out_local != NULL) { + *out_local = 0; + } + if (sublen == 1) { goto finish; } @@ -2658,6 +2794,15 @@ parse_iso_8601_date(char *str, int len, npy_datetimestruct *out) /* Time zone offset */ else if (*substr == '-' || *substr == '+') { int offset_neg = 0, offset_hour = 0, offset_minute = 0; + + /* + * Since "local" means local with respect to the current + * machine, we say this is non-local. + */ + if (out_local != NULL) { + *out_local = 0; + } + if (*substr == '-') { offset_neg = 1; } @@ -2713,24 +2858,6 @@ parse_iso_8601_date(char *str, int len, npy_datetimestruct *out) add_minutes_to_datetimestruct(out, -60 * offset_hour - offset_minute); } - /* May have a ' ' followed by an event number */ - if (sublen == 0) { - goto finish; - } - else if (sublen > 0 && *substr == ' ') { - ++substr; - --sublen; - - while (sublen > 0 && isdigit(*substr)) { - out->event = 10 * out->event + (*substr - '0'); - ++substr; - --sublen; - } - } - else { - goto parse_error; - } - /* Skip trailing whitespace */ while (sublen > 0 && isspace(*substr)) { ++substr; @@ -3293,11 +3420,15 @@ make_iso_8601_date(npy_datetimestruct *dts, char *outstr, int outlen, * datetime duck typing. The tzinfo time zone conversion would require * this style of access anyway. * + * 'out_bestunit' gives a suggested unit based on whether the object + * was a datetime.date or datetime.datetime object. + * * Returns -1 on error, 0 on success, and 1 (with no error set) * if obj doesn't have the neeeded date or datetime attributes. */ NPY_NO_EXPORT int -convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out) +convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out, + NPY_DATETIMEUNIT *out_bestunit) { PyObject *tmp; int isleap; @@ -3364,6 +3495,10 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out) !PyObject_HasAttrString(obj, "minute") || !PyObject_HasAttrString(obj, "second") || !PyObject_HasAttrString(obj, "microsecond")) { + /* The best unit for date is 'D' */ + if (out_bestunit != NULL) { + *out_bestunit = NPY_FR_D; + } return 0; } @@ -3465,6 +3600,11 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out) } } + /* The resolution of Python's datetime is 'us' */ + if (out_bestunit != NULL) { + *out_bestunit = NPY_FR_us; + } + return 0; invalid_date: @@ -3482,7 +3622,11 @@ convert_pydatetime_to_datetimestruct(PyObject *obj, npy_datetimestruct *out) } /* - * Converts a PyObject * into a datetime, in any of the forms supported + * Converts a PyObject * into a datetime, in any of the forms supported. + * + * If the units metadata isn't known ahead of time, set meta->base + * to -1, and this function will populate meta with either default + * values or values from the input object. * * Returns -1 on error, 0 on success. */ @@ -3495,6 +3639,8 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, char *str = NULL; int len = 0; npy_datetimestruct dts; + NPY_DATETIMEUNIT bestunit = -1; + /* Convert to an ASCII string for the date parser */ if (PyUnicode_Check(obj)) { bytes = PyUnicode_AsASCIIString(obj); @@ -3512,12 +3658,19 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, } /* Parse the ISO date */ - if (parse_iso_8601_date(str, len, &dts) < 0) { + if (parse_iso_8601_date(str, len, &dts, NULL, &bestunit, NULL) < 0) { Py_DECREF(bytes); return -1; } Py_DECREF(bytes); + /* Use the detected unit if none was specified */ + if (meta->base == -1) { + meta->base = bestunit; + meta->num = 1; + meta->events = 1; + } + if (convert_datetimestruct_to_datetime(meta, &dts, out) < 0) { return -1; } @@ -3526,10 +3679,22 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, } /* Do no conversion on raw integers */ else if (PyInt_Check(obj)) { + /* Use the default unit if none was specified */ + if (meta->base == -1) { + meta->base = NPY_DATETIME_DEFAULTUNIT; + meta->num = 1; + meta->events = 1; + } *out = PyInt_AS_LONG(obj); return 0; } else if (PyLong_Check(obj)) { + /* Use the default unit if none was specified */ + if (meta->base == -1) { + meta->base = NPY_DATETIME_DEFAULTUNIT; + meta->num = 1; + meta->events = 1; + } *out = PyLong_AsLongLong(obj); return 0; } @@ -3562,7 +3727,18 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, else if (PyArray_IsScalar(obj, Datetime)) { PyDatetimeScalarObject *dts = (PyDatetimeScalarObject *)obj; - return cast_datetime_to_datetime(&dts->obmeta, meta, dts->obval, out); + /* Copy the scalar directly if units weren't specified */ + if (meta->base == -1) { + *meta = dts->obmeta; + *out = dts->obval; + + return 0; + } + /* Otherwise do a casting transformation */ + else { + return cast_datetime_to_datetime(&dts->obmeta, meta, + dts->obval, out); + } } /* Datetime zero-dimensional array */ else if (PyArray_Check(obj) && @@ -3580,18 +3756,36 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, !PyArray_ISNOTSWAPPED(obj), obj); - return cast_datetime_to_datetime(obj_meta, meta, dt, out); + /* Copy the value directly if units weren't specified */ + if (meta->base == -1) { + *meta = *obj_meta; + *out = dt; + + return 0; + } + /* Otherwise do a casting transformation */ + else { + return cast_datetime_to_datetime(obj_meta, meta, dt, out); + } } /* Convert from a Python date or datetime object */ else { int code; npy_datetimestruct dts; + NPY_DATETIMEUNIT bestunit = -1; - code = convert_pydatetime_to_datetimestruct(obj, &dts); + code = convert_pydatetime_to_datetimestruct(obj, &dts, &bestunit); if (code == -1) { return -1; } else if (code == 0) { + /* Use the detected unit if none was specified */ + if (meta->base == -1) { + meta->base = bestunit; + meta->num = 1; + meta->events = 1; + } + if (convert_datetimestruct_to_datetime(meta, &dts, out) < 0) { return -1; } @@ -3608,6 +3802,10 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, /* * Converts a PyObject * into a timedelta, in any of the forms supported * + * If the units metadata isn't known ahead of time, set meta->base + * to -1, and this function will populate meta with either default + * values or values from the input object. + * * Returns -1 on error, 0 on success. */ NPY_NO_EXPORT int @@ -3616,10 +3814,24 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, { /* Do no conversion on raw integers */ if (PyInt_Check(obj)) { + /* Use the default unit if none was specified */ + if (meta->base == -1) { + meta->base = NPY_DATETIME_DEFAULTUNIT; + meta->num = 1; + meta->events = 1; + } + *out = PyInt_AS_LONG(obj); return 0; } else if (PyLong_Check(obj)) { + /* Use the default unit if none was specified */ + if (meta->base == -1) { + meta->base = NPY_DATETIME_DEFAULTUNIT; + meta->num = 1; + meta->events = 1; + } + *out = PyLong_AsLongLong(obj); return 0; } @@ -3627,8 +3839,18 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, else if (PyArray_IsScalar(obj, Timedelta)) { PyTimedeltaScalarObject *dts = (PyTimedeltaScalarObject *)obj; - return cast_timedelta_to_timedelta(&dts->obmeta, meta, - dts->obval, out); + /* Copy the scalar directly if units weren't specified */ + if (meta->base == -1) { + *meta = dts->obmeta; + *out = dts->obval; + + return 0; + } + /* Otherwise do a casting transformation */ + else { + return cast_timedelta_to_timedelta(&dts->obmeta, meta, + dts->obval, out); + } } /* Timedelta zero-dimensional array */ else if (PyArray_Check(obj) && @@ -3646,7 +3868,17 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, !PyArray_ISNOTSWAPPED(obj), obj); - return cast_timedelta_to_timedelta(obj_meta, meta, dt, out); + /* Copy the value directly if units weren't specified */ + if (meta->base == -1) { + *meta = *obj_meta; + *out = dt; + + return 0; + } + /* Otherwise do a casting transformation */ + else { + return cast_timedelta_to_timedelta(obj_meta, meta, dt, out); + } } /* Convert from a Python timedelta object */ else if (PyObject_HasAttrString(obj, "days") && @@ -3694,16 +3926,29 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, } Py_DECREF(tmp); - /* - * Convert to a microseconds timedelta, then cast to the - * desired units. - */ td = days*(24*60*60*1000000LL) + seconds*1000000LL + useconds; - us_meta.base = NPY_FR_us; - us_meta.num = 1; - us_meta.events = 1; - - return cast_timedelta_to_timedelta(&us_meta, meta, td, out); + + /* Use microseconds if none was specified */ + if (meta->base == -1) { + meta->base = NPY_FR_us; + meta->num = 1; + meta->events = 1; + + *out = td; + + return 0; + } + else { + /* + * Convert to a microseconds timedelta, then cast to the + * desired units. + */ + us_meta.base = NPY_FR_us; + us_meta.num = 1; + us_meta.events = 1; + + return cast_timedelta_to_timedelta(&us_meta, meta, td, out); + } } PyErr_SetString(PyExc_ValueError, diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index c024dccccf76..50e51e5918c9 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -2521,50 +2521,26 @@ static PyObject * } } else { - ret->obmeta.base = NPY_DATETIME_DEFAULTUNIT; - ret->obmeta.num = 1; - ret->obmeta.events = 1; + /* + * A unit of -1 signals that convert_pyobject_to_datetime + * should populate. + */ + ret->obmeta.base = -1; } if (obj == NULL) { - ret->obval = 0; - } - /* - * If the metadata was unspecified and the input is a datetime/timedelta - * scalar, then copy the input's value and metadata exactly. - */ - else if (meta_obj == NULL && PyArray_IsScalar(obj, @Name@)) { - ret->obval = ((Py@Name@ScalarObject *)obj)->obval; - ret->obmeta = ((Py@Name@ScalarObject *)obj)->obmeta; - } - /* - * If the metadata was unspecified and the input is a datetime/timedelta - * zero-dimensional array, then copy the input's value and metadata - * directly. - */ - else if (meta_obj == NULL && - PyArray_Check(obj) && - PyArray_NDIM(obj) == 0 && - PyArray_DESCR(obj)->type_num == NPY_@NAME@) { - PyArray_DatetimeMetaData *meta; - - meta = get_datetime_metadata_from_dtype(PyArray_DESCR(obj)); - if (meta == NULL) { - Py_DECREF(ret); - return NULL; + if (ret->obmeta.base == -1) { + ret->obmeta.base = NPY_DATETIME_DEFAULTUNIT; + ret->obmeta.num = 1; + ret->obmeta.events = 1; } - PyArray_DESCR(obj)->f->copyswap(&ret->obval, - PyArray_DATA(obj), - !PyArray_ISNOTSWAPPED(obj), - obj); - ret->obmeta = *meta; + + ret->obval = 0; } - else { - if (convert_pyobject_to_@name@(&ret->obmeta, obj, &ret->obval) + else if (convert_pyobject_to_@name@(&ret->obmeta, obj, &ret->obval) < 0) { - Py_DECREF(ret); - return NULL; - } + Py_DECREF(ret); + return NULL; } return (PyObject *)ret; diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index bb6594a9a95c..9fb249340776 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -80,6 +80,9 @@ def test_datetime_scalar_construction(self): assert_equal(np.datetime64('1950-03-12', 's'), np.datetime64('1950-03-12', 'm')) + # Default construction means 0 + assert_equal(np.datetime64(), np.datetime64(0)) + # When constructing from a scalar or zero-dimensional array, # it either keeps the units or you can override them. a = np.datetime64('2000-03-18T16Z', 'h') @@ -110,7 +113,6 @@ def test_datetime_scalar_construction(self): np.datetime64(datetime.datetime(1980,1,25, 14,36,22,500000))) - def test_timedelta_scalar_construction(self): # Construct with different units assert_equal(np.timedelta64(7, 'D'), @@ -118,6 +120,9 @@ def test_timedelta_scalar_construction(self): assert_equal(np.timedelta64(120, 's'), np.timedelta64(2, 'm')) + # Default construction means 0 + assert_equal(np.timedelta64(), np.timedelta64(0)) + # When constructing from a scalar or zero-dimensional array, # it either keeps the units or you can override them. a = np.timedelta64(2, 'h') @@ -158,6 +163,86 @@ def test_timedelta_scalar_construction(self): assert_equal(np.timedelta64(28, 'W'), np.timedelta64(datetime.timedelta(weeks=28))) + def test_timedelta_scalar_construction_units(self): + # String construction detecting units + assert_equal(np.datetime64('2010').dtype, + np.dtype('M8[Y]')) + assert_equal(np.datetime64('2010-03').dtype, + np.dtype('M8[M]')) + assert_equal(np.datetime64('2010-03-12').dtype, + np.dtype('M8[D]')) + assert_equal(np.datetime64('2010-03-12T17').dtype, + np.dtype('M8[h]')) + assert_equal(np.datetime64('2010-03-12T17:15Z').dtype, + np.dtype('M8[m]')) + assert_equal(np.datetime64('2010-03-12T17:15:08Z').dtype, + np.dtype('M8[s]')) + + assert_equal(np.datetime64('2010-03-12T17:15:08.1Z').dtype, + np.dtype('M8[ms]')) + assert_equal(np.datetime64('2010-03-12T17:15:08.12Z').dtype, + np.dtype('M8[ms]')) + assert_equal(np.datetime64('2010-03-12T17:15:08.123Z').dtype, + np.dtype('M8[ms]')) + + assert_equal(np.datetime64('2010-03-12T17:15:08.1234Z').dtype, + np.dtype('M8[us]')) + assert_equal(np.datetime64('2010-03-12T17:15:08.12345Z').dtype, + np.dtype('M8[us]')) + assert_equal(np.datetime64('2010-03-12T17:15:08.123456Z').dtype, + np.dtype('M8[us]')) + + assert_equal(np.datetime64('1970-01-01T00:00:02.1234567Z').dtype, + np.dtype('M8[ns]')) + assert_equal(np.datetime64('1970-01-01T00:00:02.12345678Z').dtype, + np.dtype('M8[ns]')) + assert_equal(np.datetime64('1970-01-01T00:00:02.123456789Z').dtype, + np.dtype('M8[ns]')) + + assert_equal(np.datetime64('1970-01-01T00:00:02.1234567890Z').dtype, + np.dtype('M8[ps]')) + assert_equal(np.datetime64('1970-01-01T00:00:02.12345678901Z').dtype, + np.dtype('M8[ps]')) + assert_equal(np.datetime64('1970-01-01T00:00:02.123456789012Z').dtype, + np.dtype('M8[ps]')) + + assert_equal(np.datetime64( + '1970-01-01T00:00:02.1234567890123Z').dtype, + np.dtype('M8[fs]')) + assert_equal(np.datetime64( + '1970-01-01T00:00:02.12345678901234Z').dtype, + np.dtype('M8[fs]')) + assert_equal(np.datetime64( + '1970-01-01T00:00:02.123456789012345Z').dtype, + np.dtype('M8[fs]')) + + assert_equal(np.datetime64( + '1970-01-01T00:00:02.1234567890123456Z').dtype, + np.dtype('M8[as]')) + assert_equal(np.datetime64( + '1970-01-01T00:00:02.12345678901234567Z').dtype, + np.dtype('M8[as]')) + assert_equal(np.datetime64( + '1970-01-01T00:00:02.123456789012345678Z').dtype, + np.dtype('M8[as]')) + + # Python date object + assert_equal(np.datetime64(datetime.date(2010,4,16)).dtype, + np.dtype('M8[D]')) + + # Python datetime object + assert_equal(np.datetime64( + datetime.datetime(2010,4,16,13,45,18)).dtype, + np.dtype('M8[us]')) + + # 'today' special value + assert_equal(np.datetime64('today').dtype, + np.dtype('M8[D]')) + + # 'now' special value + assert_equal(np.datetime64('now').dtype, + np.dtype('M8[s]')) + def test_datetime_nat_casting(self): a = np.array('NaT', dtype='M8[D]') b = np.datetime64('NaT', '[D]') From 6395979c7bb17dc79b2037884c383b46b75bb67f Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Tue, 7 Jun 2011 15:37:26 -0500 Subject: [PATCH 02/34] ENH: datetime-autounit: Make 'now' and 'today' only parse with appropriate units In particular, 'now' needs time-like units, and 'today' needs date-like units. --- numpy/core/src/multiarray/_datetime.h | 4 +++- numpy/core/src/multiarray/datetime.c | 23 ++++++++++++++++++++++- numpy/core/tests/test_datetime.py | 14 +++++++++----- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/numpy/core/src/multiarray/_datetime.h b/numpy/core/src/multiarray/_datetime.h index b07fe2f514f6..643c9b106cdb 100644 --- a/numpy/core/src/multiarray/_datetime.h +++ b/numpy/core/src/multiarray/_datetime.h @@ -192,6 +192,8 @@ get_datetime_iso_8601_strlen(int local, NPY_DATETIMEUNIT base); * day according to local time) and "Now" (current time in UTC). * * 'str' must be a NULL-terminated string, and 'len' must be its length. + * 'unit' should contain -1 if the unit is unknown, or the unit + * which will be used if it is. * * 'out' gets filled with the parsed date-time. * 'out_local' gets set to 1 if the parsed time was in local time, @@ -210,12 +212,12 @@ get_datetime_iso_8601_strlen(int local, NPY_DATETIMEUNIT base); */ NPY_NO_EXPORT int parse_iso_8601_date(char *str, int len, + NPY_DATETIMEUNIT unit, npy_datetimestruct *out, npy_bool *out_local, NPY_DATETIMEUNIT *out_bestunit, npy_bool *out_special); - /* * Converts an npy_datetimestruct to an (almost) ISO 8601 * NULL-terminated string. diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 8e57961eb85c..642341570af6 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -2278,6 +2278,8 @@ add_minutes_to_datetimestruct(npy_datetimestruct *dts, int minutes) * day according to local time) and "Now" (current time in UTC). * * 'str' must be a NULL-terminated string, and 'len' must be its length. + * 'unit' should contain -1 if the unit is unknown, or the unit + * which will be used if it is. * * 'out' gets filled with the parsed date-time. * 'out_local' gets set to 1 if the parsed time was in local time, @@ -2296,6 +2298,7 @@ add_minutes_to_datetimestruct(npy_datetimestruct *dts, int minutes) */ NPY_NO_EXPORT int parse_iso_8601_date(char *str, int len, + NPY_DATETIMEUNIT unit, npy_datetimestruct *out, npy_bool *out_local, NPY_DATETIMEUNIT *out_bestunit, @@ -2350,6 +2353,14 @@ parse_iso_8601_date(char *str, int len, time_t rawtime = 0; struct tm tm_; + /* 'today' only works for units of days or larger */ + if (unit != -1 && unit > NPY_FR_D) { + PyErr_SetString(PyExc_ValueError, + "Special value 'today' can only be converted " + "to a NumPy datetime with 'D' or larger units"); + return -1; + } + time(&rawtime); #if defined(_WIN32) if (localtime_s(&tm_, &rawtime) != 0) { @@ -2392,6 +2403,15 @@ parse_iso_8601_date(char *str, int len, tolower(str[2]) == 'w') { time_t rawtime = 0; PyArray_DatetimeMetaData meta; + + /* 'now' only works for units of hours or smaller */ + if (unit != -1 && unit < NPY_FR_h) { + PyErr_SetString(PyExc_ValueError, + "Special value 'now' can only be converted " + "to a NumPy datetime with 'h' or smaller units"); + return -1; + } + time(&rawtime); /* Set up a dummy metadata for the conversion */ @@ -3658,7 +3678,8 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, } /* Parse the ISO date */ - if (parse_iso_8601_date(str, len, &dts, NULL, &bestunit, NULL) < 0) { + if (parse_iso_8601_date(str, len, meta->base, &dts, + NULL, &bestunit, NULL) < 0) { Py_DECREF(bytes); return -1; } diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index 9fb249340776..316d03ecc823 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -239,10 +239,18 @@ def test_timedelta_scalar_construction_units(self): assert_equal(np.datetime64('today').dtype, np.dtype('M8[D]')) + assert_raises(ValueError, np.datetime64, 'today', 'h') + assert_raises(ValueError, np.datetime64, 'today', 's') + assert_raises(ValueError, np.datetime64, 'today', 'as') + # 'now' special value assert_equal(np.datetime64('now').dtype, np.dtype('M8[s]')) + assert_raises(ValueError, np.datetime64, 'now', 'Y') + assert_raises(ValueError, np.datetime64, 'now', 'M') + assert_raises(ValueError, np.datetime64, 'now', 'D') + def test_datetime_nat_casting(self): a = np.array('NaT', dtype='M8[D]') b = np.datetime64('NaT', '[D]') @@ -352,7 +360,7 @@ def test_pydatetime_creation(self): a = np.array(['2000-01-01', datetime.date(2000, 1, 1)], dtype='M8[s]') assert_equal(a[0], a[1]) # Will fail if the date changes during the exact right moment - a = np.array(['today', datetime.date.today()], dtype='M8[s]') + a = np.array(['today', datetime.date.today()], dtype='M8[D]') assert_equal(a[0], a[1]) # datetime.datetime.now() returns local time, not UTC #a = np.array(['now', datetime.datetime.now()], dtype='M8[s]') @@ -470,14 +478,10 @@ def test_different_unit_comparison(self): np.array('1932-02-17T00:00:00Z', dtype=dt2)) assert_equal(np.array('10000-04-27', dtype=dt1), np.array('10000-04-27T00:00:00Z', dtype=dt2)) - assert_equal(np.array('today', dtype=dt1), - np.array('today', dtype=dt2)) assert_equal(np.datetime64('1932-02-17', unit1), np.datetime64('1932-02-17T00:00:00Z', unit2)) assert_equal(np.datetime64('10000-04-27', unit1), np.datetime64('10000-04-27T00:00:00Z', unit2)) - assert_equal(np.datetime64('today', unit1), - np.datetime64('today', unit2)) # Shouldn't be able to compare datetime and timedelta # TODO: Changing to 'same_kind' or 'safe' casting in the ufuncs by From ab2ac7ca591256e25181ba600b23a0dd155cc1c2 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Tue, 7 Jun 2011 16:37:01 -0500 Subject: [PATCH 03/34] ENH: datetime-promotion: Unify datetime/timedelta type promotion Now it always goes to the more precise unit. --- numpy/core/src/multiarray/_datetime.h | 13 ---- numpy/core/src/multiarray/datetime.c | 88 +++++++++++----------- numpy/core/src/umath/ufunc_object.c | 102 +++++++++++++++----------- numpy/core/tests/test_datetime.py | 38 +++++----- 4 files changed, 119 insertions(+), 122 deletions(-) diff --git a/numpy/core/src/multiarray/_datetime.h b/numpy/core/src/multiarray/_datetime.h index 643c9b106cdb..05dab36a45ad 100644 --- a/numpy/core/src/multiarray/_datetime.h +++ b/numpy/core/src/multiarray/_datetime.h @@ -93,19 +93,6 @@ datetime_metadata_divides( PyArray_Descr *divisor, int strict_with_nonlinear_units); -/* - * Computes the GCD of the two date-time metadata values. Raises - * an exception if there is no reasonable GCD, such as with - * years and days. - * - * Returns a capsule with the GCD metadata. - */ -NPY_NO_EXPORT PyObject * -compute_datetime_metadata_greatest_common_divisor( - PyArray_Descr *type1, - PyArray_Descr *type2, - int strict_with_nonlinear_units); - /* * Computes the conversion factor to convert data with 'src_meta' metadata * into data with 'dst_meta' metadata, not taking into account the events. diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 642341570af6..1a85209b5687 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -1632,11 +1632,12 @@ datetime_metadata_divides( } -NPY_NO_EXPORT PyObject * +static PyObject * compute_datetime_metadata_greatest_common_divisor( PyArray_Descr *type1, PyArray_Descr *type2, - int strict_with_nonlinear_units) + int strict_with_nonlinear_units1, + int strict_with_nonlinear_units2) { PyArray_DatetimeMetaData *meta1, *meta2, *dt_data; NPY_DATETIMEUNIT base; @@ -1688,7 +1689,7 @@ compute_datetime_metadata_greatest_common_divisor( base = NPY_FR_M; num1 *= 12; } - else if (strict_with_nonlinear_units) { + else if (strict_with_nonlinear_units1) { goto incompatible_units; } else { @@ -1701,7 +1702,7 @@ compute_datetime_metadata_greatest_common_divisor( base = NPY_FR_M; num2 *= 12; } - else if (strict_with_nonlinear_units) { + else if (strict_with_nonlinear_units2) { goto incompatible_units; } else { @@ -1709,11 +1710,26 @@ compute_datetime_metadata_greatest_common_divisor( /* Don't multiply num2 since there is no even factor */ } } - else if (meta1->base == NPY_FR_M || - meta1->base == NPY_FR_B || - meta2->base == NPY_FR_M || - meta2->base == NPY_FR_B) { - if (strict_with_nonlinear_units) { + else if (meta1->base == NPY_FR_M) { + if (strict_with_nonlinear_units1) { + goto incompatible_units; + } + else { + base = meta2->base; + /* Don't multiply num1 since there is no even factor */ + } + } + else if (meta2->base == NPY_FR_M) { + if (strict_with_nonlinear_units2) { + goto incompatible_units; + } + else { + base = meta1->base; + /* Don't multiply num2 since there is no even factor */ + } + } + else if (meta1->base == NPY_FR_B || meta2->base == NPY_FR_B) { + if (strict_with_nonlinear_units1 || strict_with_nonlinear_units2) { goto incompatible_units; } else { @@ -1801,14 +1817,22 @@ units_overflow: { } /* - * Uses type1's type_num and the gcd of the metadata to create - * the result type. + * Both type1 and type2 must be either NPY_DATETIME or NPY_TIMEDELTA. + * Applies the type promotion rules between the two types, returning + * the promoted type. */ -static PyArray_Descr * -datetime_gcd_type_promotion(PyArray_Descr *type1, PyArray_Descr *type2) +NPY_NO_EXPORT PyArray_Descr * +datetime_type_promotion(PyArray_Descr *type1, PyArray_Descr *type2) { + int type_num1, type_num2; PyObject *gcdmeta; PyArray_Descr *dtype; + int is_datetime; + + type_num1 = type1->type_num; + type_num2 = type2->type_num; + + is_datetime = (type_num1 == NPY_DATETIME || type_num2 == NPY_DATETIME); /* * Get the metadata GCD, being strict about nonlinear units for @@ -1816,13 +1840,15 @@ datetime_gcd_type_promotion(PyArray_Descr *type1, PyArray_Descr *type2) */ gcdmeta = compute_datetime_metadata_greatest_common_divisor( type1, type2, - type1->type_num == NPY_TIMEDELTA); + type_num1 == NPY_TIMEDELTA, + type_num2 == NPY_TIMEDELTA); if (gcdmeta == NULL) { return NULL; } /* Create a DATETIME or TIMEDELTA dtype */ - dtype = PyArray_DescrNewFromType(type1->type_num); + dtype = PyArray_DescrNewFromType(is_datetime ? NPY_DATETIME : + NPY_TIMEDELTA); if (dtype == NULL) { Py_DECREF(gcdmeta); return NULL; @@ -1847,39 +1873,7 @@ datetime_gcd_type_promotion(PyArray_Descr *type1, PyArray_Descr *type2) Py_DECREF(gcdmeta); return dtype; -} - -/* - * Both type1 and type2 must be either NPY_DATETIME or NPY_TIMEDELTA. - * Applies the type promotion rules between the two types, returning - * the promoted type. - */ -NPY_NO_EXPORT PyArray_Descr * -datetime_type_promotion(PyArray_Descr *type1, PyArray_Descr *type2) -{ - int type_num1, type_num2; - - type_num1 = type1->type_num; - type_num2 = type2->type_num; - if (type_num1 == NPY_DATETIME) { - if (type_num2 == NPY_DATETIME) { - return datetime_gcd_type_promotion(type1, type2); - } - else if (type_num2 == NPY_TIMEDELTA) { - Py_INCREF(type1); - return type1; - } - } - else if (type_num1 == NPY_TIMEDELTA) { - if (type_num2 == NPY_DATETIME) { - Py_INCREF(type2); - return type2; - } - else if (type_num2 == NPY_TIMEDELTA) { - return datetime_gcd_type_promotion(type1, type2); - } - } PyErr_SetString(PyExc_RuntimeError, "Called datetime_type_promotion on non-datetype type"); diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 009876bab457..e37dd0167046 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -2243,8 +2243,8 @@ timedelta_dtype_with_copied_meta(PyArray_Descr *dtype) * int + m8[] => m8[] + m8[] * M8[] + int => M8[] + m8[] * int + M8[] => m8[] + M8[] - * M8[] + m8[] => M8[] + m8[] - * m8[] + M8[] => m8[] + M8[] + * M8[] + m8[] => M8[gcd(,)] + m8[gcd(,)] + * m8[] + M8[] => m8[gcd(,)] + M8[gcd(,)] * TODO: Non-linear time unit cases require highly special-cased loops * M8[] + m8[Y|M|B] * m8[Y|M|B] + M8[] @@ -2287,16 +2287,20 @@ PyUFunc_AdditionTypeResolution(PyUFuncObject *ufunc, out_dtypes[2] = out_dtypes[0]; Py_INCREF(out_dtypes[2]); } - /* m8[] + M8[] => m8[] + M8[] */ + /* m8[] + M8[] => m8[gcd(,)] + M8[gcd(,)] */ else if (type_num2 == NPY_DATETIME) { - /* Make a new NPY_TIMEDELTA, and copy type2's metadata */ - out_dtypes[0] = timedelta_dtype_with_copied_meta( - PyArray_DESCR(operands[1])); + out_dtypes[1] = PyArray_PromoteTypes(PyArray_DESCR(operands[0]), + PyArray_DESCR(operands[1])); + if (out_dtypes[1] == NULL) { + return -1; + } + /* Make a new NPY_TIMEDELTA, and copy the datetime's metadata */ + out_dtypes[0] = timedelta_dtype_with_copied_meta(out_dtypes[1]); if (out_dtypes[0] == NULL) { + Py_DECREF(out_dtypes[1]); + out_dtypes[1] = NULL; return -1; } - out_dtypes[1] = PyArray_DESCR(operands[1]); - Py_INCREF(out_dtypes[1]); out_dtypes[2] = out_dtypes[1]; Py_INCREF(out_dtypes[2]); } @@ -2317,10 +2321,25 @@ PyUFunc_AdditionTypeResolution(PyUFuncObject *ufunc, } } else if (type_num1 == NPY_DATETIME) { - /* M8[] + m8[] => M8[] + m8[] */ + /* M8[] + m8[] => M8[gcd(,)] + m8[gcd(,)] */ + if (type_num2 == NPY_TIMEDELTA) { + out_dtypes[0] = PyArray_PromoteTypes(PyArray_DESCR(operands[0]), + PyArray_DESCR(operands[1])); + if (out_dtypes[0] == NULL) { + return -1; + } + /* Make a new NPY_TIMEDELTA, and copy the datetime's metadata */ + out_dtypes[1] = timedelta_dtype_with_copied_meta(out_dtypes[0]); + if (out_dtypes[1] == NULL) { + Py_DECREF(out_dtypes[0]); + out_dtypes[0] = NULL; + return -1; + } + out_dtypes[2] = out_dtypes[0]; + Py_INCREF(out_dtypes[2]); + } /* M8[] + int => M8[] + m8[] */ - if (type_num2 == NPY_TIMEDELTA || - PyTypeNum_ISINTEGER(type_num2) || + else if (PyTypeNum_ISINTEGER(type_num2) || PyTypeNum_ISBOOL(type_num2)) { /* Make a new NPY_TIMEDELTA, and copy type1's metadata */ out_dtypes[1] = timedelta_dtype_with_copied_meta( @@ -2421,7 +2440,7 @@ type_reso_error: { * m8[] - int => m8[] - m8[] * int - m8[] => m8[] - m8[] * M8[] - int => M8[] - m8[] - * M8[] - m8[] => M8[] - m8[] + * M8[] - m8[] => M8[gcd(,)] - m8[gcd(,)] * TODO: Non-linear time unit cases require highly special-cased loops * M8[] - m8[Y|M|B] */ @@ -2480,10 +2499,25 @@ PyUFunc_SubtractionTypeResolution(PyUFuncObject *ufunc, } } else if (type_num1 == NPY_DATETIME) { - /* M8[] - m8[] => M8[] - m8[] */ + /* M8[] - m8[] => M8[gcd(,)] - m8[gcd(,)] */ + if (type_num2 == NPY_TIMEDELTA) { + out_dtypes[0] = PyArray_PromoteTypes(PyArray_DESCR(operands[0]), + PyArray_DESCR(operands[1])); + if (out_dtypes[0] == NULL) { + return -1; + } + /* Make a new NPY_TIMEDELTA, and copy the datetime's metadata */ + out_dtypes[1] = timedelta_dtype_with_copied_meta(out_dtypes[0]); + if (out_dtypes[1] == NULL) { + Py_DECREF(out_dtypes[0]); + out_dtypes[0] = NULL; + return -1; + } + out_dtypes[2] = out_dtypes[0]; + Py_INCREF(out_dtypes[2]); + } /* M8[] - int => M8[] - m8[] */ - if (type_num2 == NPY_TIMEDELTA || - PyTypeNum_ISINTEGER(type_num2) || + else if (PyTypeNum_ISINTEGER(type_num2) || PyTypeNum_ISBOOL(type_num2)) { /* Make a new NPY_TIMEDELTA, and copy type1's metadata */ out_dtypes[1] = timedelta_dtype_with_copied_meta( @@ -2498,39 +2532,21 @@ PyUFunc_SubtractionTypeResolution(PyUFuncObject *ufunc, type_num2 = NPY_TIMEDELTA; } - /* M8[] - M8[] (producing m8[])*/ + /* M8[] - M8[] => M8[gcd(,)] - M8[gcd(,)] */ else if (type_num2 == NPY_DATETIME) { - PyArray_DatetimeMetaData *meta1, *meta2; - - meta1 = get_datetime_metadata_from_dtype( - PyArray_DESCR(operands[0])); - if (meta1 == NULL) { + out_dtypes[0] = PyArray_PromoteTypes(PyArray_DESCR(operands[0]), + PyArray_DESCR(operands[1])); + if (out_dtypes[0] == NULL) { return -1; } - meta2 = get_datetime_metadata_from_dtype( - PyArray_DESCR(operands[1])); - if (meta2 == NULL) { + /* Make a new NPY_TIMEDELTA, and copy type1's metadata */ + out_dtypes[2] = timedelta_dtype_with_copied_meta(out_dtypes[0]); + if (out_dtypes[2] == NULL) { + Py_DECREF(out_dtypes[0]); return -1; } - - /* If the metadata matches up, the subtraction is ok */ - if (meta1->num == meta2->num && - meta1->base == meta2->base && - meta1->events == meta2->events) { - out_dtypes[0] = PyArray_DESCR(operands[1]); - Py_INCREF(out_dtypes[0]); - out_dtypes[1] = out_dtypes[0]; - Py_INCREF(out_dtypes[1]); - /* Make a new NPY_TIMEDELTA, and copy type1's metadata */ - out_dtypes[2] = timedelta_dtype_with_copied_meta( - PyArray_DESCR(operands[0])); - if (out_dtypes[2] == NULL) { - return -1; - } - } - else { - goto type_reso_error; - } + out_dtypes[1] = out_dtypes[0]; + Py_INCREF(out_dtypes[1]); } else { goto type_reso_error; diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index 316d03ecc823..994066137a91 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -539,7 +539,7 @@ def test_datetime_add(self): # One-dimensional arrays (np.array(['2012-12-21'], dtype='M8[D]'), np.array(['2012-12-24'], dtype='M8[D]'), - np.array(['1940-12-24'], dtype='M8[D]'), + np.array(['2012-12-21T11Z'], dtype='M8[h]'), np.array(['NaT'], dtype='M8[D]'), np.array([3], dtype='m8[D]'), np.array([11], dtype='m8[h]'), @@ -547,7 +547,7 @@ def test_datetime_add(self): # NumPy scalars (np.datetime64('2012-12-21', '[D]'), np.datetime64('2012-12-24', '[D]'), - np.datetime64('1940-12-24', '[D]'), + np.datetime64('2012-12-21T11Z', '[h]'), np.datetime64('NaT', '[D]'), np.timedelta64(3, '[D]'), np.timedelta64(11, '[h]'), @@ -592,27 +592,24 @@ def test_datetime_add(self): assert_equal(tda + dtnat, dtnat) assert_equal((tda + dta).dtype, np.dtype('M8[D]')) - # In M8 + m8, the M8 controls the result type - assert_equal(dta + tdb, dta) - assert_equal((dta + tdb).dtype, np.dtype('M8[D]')) - assert_equal(dtc + tdb, dtc) - assert_equal((dtc + tdb).dtype, np.dtype('M8[D]')) - assert_equal(tdb + dta, dta) - assert_equal((tdb + dta).dtype, np.dtype('M8[D]')) - assert_equal(tdb + dtc, dtc) - assert_equal((tdb + dtc).dtype, np.dtype('M8[D]')) + # In M8 + m8, the result goes to higher precision + assert_equal(dta + tdb, dtc) + assert_equal((dta + tdb).dtype, np.dtype('M8[h]')) + assert_equal(tdb + dta, dtc) + assert_equal((tdb + dta).dtype, np.dtype('M8[h]')) # M8 + M8 assert_raises(TypeError, np.add, dta, dtb) def test_datetime_subtract(self): - for dta, dtb, dtc, dtd, dtnat, tda, tdb, tdc in \ + for dta, dtb, dtc, dtd, dte, dtnat, tda, tdb, tdc in \ [ # One-dimensional arrays (np.array(['2012-12-21'], dtype='M8[D]'), np.array(['2012-12-24'], dtype='M8[D]'), np.array(['1940-12-24'], dtype='M8[D]'), np.array(['1940-12-24'], dtype='M8[h]'), + np.array(['1940-12-23T13Z'], dtype='M8[h]'), np.array(['NaT'], dtype='M8[D]'), np.array([3], dtype='m8[D]'), np.array([11], dtype='m8[h]'), @@ -622,6 +619,7 @@ def test_datetime_subtract(self): np.datetime64('2012-12-24', '[D]'), np.datetime64('1940-12-24', '[D]'), np.datetime64('1940-12-24', '[h]'), + np.datetime64('1940-12-23T13Z', '[h]'), np.datetime64('NaT', '[D]'), np.timedelta64(3, '[D]'), np.timedelta64(11, '[h]'), @@ -656,14 +654,16 @@ def test_datetime_subtract(self): assert_equal(dtnat - tda, dtnat) assert_equal((dtb - tda).dtype, np.dtype('M8[D]')) - # In M8 - m8, the M8 controls the result type - assert_equal(dta - tdb, dta) - assert_equal((dta - tdb).dtype, np.dtype('M8[D]')) - assert_equal(dtc - tdb, dtc) - assert_equal((dtc - tdb).dtype, np.dtype('M8[D]')) + # In M8 - m8, the result goes to higher precision + assert_equal(dtc - tdb, dte) + assert_equal((dtc - tdb).dtype, np.dtype('M8[h]')) + + # M8 - M8 with different goes to higher precision + assert_equal(dtc - dtd, 0) + assert_equal((dtc - dtd).dtype, np.dtype('m8[h]')) + assert_equal(dtd - dtc, 0) + assert_equal((dtd - dtc).dtype, np.dtype('m8[h]')) - # M8 - M8 with different metadata - assert_raises(TypeError, np.subtract, dtc, dtd) # m8 - M8 assert_raises(TypeError, np.subtract, tda, dta) # bool - M8 From fdb4190225df2cd4f1d966f4086372f0a866e54a Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Wed, 8 Jun 2011 16:41:21 -0500 Subject: [PATCH 04/34] BUG: datetime: Had int instead of Py_ssize_t for an AsStringAndSize call --- numpy/core/src/multiarray/datetime.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 1a85209b5687..17733fc1c962 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -3651,7 +3651,7 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, if (PyBytes_Check(obj) || PyUnicode_Check(obj)) { PyObject *bytes = NULL; char *str = NULL; - int len = 0; + Py_ssize_t len = 0; npy_datetimestruct dts; NPY_DATETIMEUNIT bestunit = -1; From 487874afd0c8ef3388d39c803cebb9dd8996a28e Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Wed, 8 Jun 2011 11:44:37 -0500 Subject: [PATCH 05/34] ENH: datetime-arange: Add boilerplate for the specialized datetime_arange --- numpy/core/src/multiarray/_datetime.h | 14 +++++ numpy/core/src/multiarray/ctors.c | 9 ++++ numpy/core/src/multiarray/datetime.c | 73 +++++++++++++++++++++++++++ 3 files changed, 96 insertions(+) diff --git a/numpy/core/src/multiarray/_datetime.h b/numpy/core/src/multiarray/_datetime.h index 05dab36a45ad..20e40eec56e3 100644 --- a/numpy/core/src/multiarray/_datetime.h +++ b/numpy/core/src/multiarray/_datetime.h @@ -352,4 +352,18 @@ cast_timedelta_to_timedelta(PyArray_DatetimeMetaData *src_meta, npy_timedelta src_dt, npy_timedelta *dst_dt); +/* + * Returns true if the object is something that is best considered + * a Datetime or Timedelta, false otherwise. + */ +NPY_NO_EXPORT npy_bool +is_any_numpy_datetime_or_timedelta(PyObject *obj); + +/* + * Implements a datetime-specific arange + */ +NPY_NO_EXPORT PyObject * +datetime_arange(PyObject *start, PyObject *stop, PyObject *step, + PyArray_Descr *dtype); + #endif diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index a292a1a82592..f1bc7bcefeb4 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -3083,6 +3083,15 @@ PyArray_ArangeObj(PyObject *start, PyObject *stop, PyObject *step, PyArray_Descr PyArray_Descr *native = NULL; int swap; + /* Datetime arange is handled specially */ + if ((dtype != NULL && (dtype->type_num == NPY_DATETIME || + dtype->type_num == NPY_TIMEDELTA)) || + (dtype == NULL && is_any_numpy_datetime_or_timedelta(start) || + is_any_numpy_datetime_or_timedelta(stop) || + is_any_numpy_datetime_or_timedelta(step))) { + return datetime_arange(start, stop, step, dtype); + } + if (!dtype) { PyArray_Descr *deftype; PyArray_Descr *newtype; diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 17733fc1c962..3d0a9658a8b7 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -4295,3 +4295,76 @@ cast_timedelta_to_timedelta(PyArray_DatetimeMetaData *src_meta, return 0; } +/* + * Returns true if the object is something that is best considered + * a Datetime, false otherwise. + */ +static npy_bool +is_any_numpy_datetime(PyObject *obj) +{ + return (PyArray_IsScalar(obj, Datetime) || + (PyArray_Check(obj) && ( + PyArray_DESCR(obj)->type_num == NPY_DATETIME)) || + PyDate_Check(obj) || + PyDatetime_Check(obj)); +} + +/* + * Returns true if the object is something that is best considered + * a Timedelta, false otherwise. + */ +static npy_bool +is_any_numpy_timedelta(PyObject *obj) +{ + return (PyArray_IsScalar(obj, Timedelta) || + (PyArray_Check(obj) && ( + PyArray_DESCR(obj)->type_num == NPY_TIMEDELTA)) || + PyDelta_Check(obj)); +} + +/* + * Returns true if the object is something that is best considered + * a Datetime or Timedelta, false otherwise. + */ +NPY_NO_EXPORT npy_bool +is_any_numpy_datetime_or_timedelta(PyObject *obj) +{ + return obj != NULL && + (is_any_numpy_datetime(obj) || + is_any_numpy_timedelta(obj)); +} + +NPY_NO_EXPORT PyObject * +datetime_arange(PyObject *start, PyObject *stop, PyObject *step, + PyArray_Descr *dtype) +{ + PyArray_DatetimeMetaData meta_start, meta_stop, meta_step; + /* + * First normalize the input parameters so there is no Py_None, + * and start is moved to stop if stop is unspecified. + */ + if (step == Py_None) { + step = NULL; + } + if (stop == NULL || stop == Py_None) { + stop = start; + start = NULL; + /* If start was NULL or None, raise an exception */ + if (stop == NULL || stop == Py_None) { + PyErr_SetString(PyExc_ValueError, + "arange needs at least a stopping value"); + return NULL; + } + } + if (start == Py_None) { + start = NULL; + } + + /* + * Now figure out whether we're doing a datetime or a timedelta + * arange, and extract the values. + */ + + PyErr_SetString(PyExc_RuntimeError, "datetime_arange is incomplete"); + return NULL; +} From 50261be60dba090c65ee88c1531fb00a532db17a Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Wed, 8 Jun 2011 15:51:00 -0500 Subject: [PATCH 06/34] ENH: datetime-arange: Filling in the datetime-specific arange function Here I've realized that default 'microsecond' units isn't very good, and would like to make a default 'generic' unit instead. --- numpy/core/src/multiarray/_datetime.h | 14 + numpy/core/src/multiarray/ctors.c | 10 +- numpy/core/src/multiarray/datetime.c | 414 ++++++++++++++++++++++---- 3 files changed, 365 insertions(+), 73 deletions(-) diff --git a/numpy/core/src/multiarray/_datetime.h b/numpy/core/src/multiarray/_datetime.h index 20e40eec56e3..34b3b5eba75f 100644 --- a/numpy/core/src/multiarray/_datetime.h +++ b/numpy/core/src/multiarray/_datetime.h @@ -93,6 +93,20 @@ datetime_metadata_divides( PyArray_Descr *divisor, int strict_with_nonlinear_units); +/* + * Computes the GCD of the two date-time metadata values. Raises + * an exception if there is no reasonable GCD, such as with + * years and days. + * + * Returns a capsule with the GCD metadata. + */ +NPY_NO_EXPORT PyObject * +compute_datetime_metadata_greatest_common_divisor_capsule( + PyArray_Descr *type1, + PyArray_Descr *type2, + int strict_with_nonlinear_units1, + int strict_with_nonlinear_units2); + /* * Computes the conversion factor to convert data with 'src_meta' metadata * into data with 'dst_meta' metadata, not taking into account the events. diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index f1bc7bcefeb4..b400f1feccab 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -14,16 +14,12 @@ #include "numpy/npy_3kcompat.h" #include "common.h" - #include "ctors.h" - #include "shape.h" - #include "buffer.h" - #include "numpymemoryview.h" - #include "lowlevel_strided_loops.h" +#include "_datetime.h" /* * Reading from a file or a string. @@ -3086,9 +3082,9 @@ PyArray_ArangeObj(PyObject *start, PyObject *stop, PyObject *step, PyArray_Descr /* Datetime arange is handled specially */ if ((dtype != NULL && (dtype->type_num == NPY_DATETIME || dtype->type_num == NPY_TIMEDELTA)) || - (dtype == NULL && is_any_numpy_datetime_or_timedelta(start) || + (dtype == NULL && (is_any_numpy_datetime_or_timedelta(start) || is_any_numpy_datetime_or_timedelta(stop) || - is_any_numpy_datetime_or_timedelta(step))) { + is_any_numpy_datetime_or_timedelta(step)))) { return datetime_arange(start, stop, step, dtype); } diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 3d0a9658a8b7..a8970dc7bab2 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -1631,38 +1631,27 @@ datetime_metadata_divides( return (num1 % num2) == 0; } - -static PyObject * +/* + * Computes the GCD of the two date-time metadata values. Raises + * an exception if there is no reasonable GCD, such as with + * years and days. + * + * The result is placed in 'out_meta'. + * + * Returns 0 on success, -1 on failure. + */ +NPY_NO_EXPORT int compute_datetime_metadata_greatest_common_divisor( - PyArray_Descr *type1, - PyArray_Descr *type2, + PyArray_DatetimeMetaData *meta1, + PyArray_DatetimeMetaData *meta2, + PyArray_DatetimeMetaData *out_meta, int strict_with_nonlinear_units1, int strict_with_nonlinear_units2) { - PyArray_DatetimeMetaData *meta1, *meta2, *dt_data; NPY_DATETIMEUNIT base; npy_uint64 num1, num2, num; int events = 1; - if ((type1->type_num != NPY_DATETIME && - type1->type_num != NPY_TIMEDELTA) || - (type2->type_num != NPY_DATETIME && - type2->type_num != NPY_TIMEDELTA)) { - PyErr_SetString(PyExc_TypeError, - "Require datetime types for metadata " - "greatest common divisor operation"); - return NULL; - } - - meta1 = get_datetime_metadata_from_dtype(type1); - if (meta1 == NULL) { - return NULL; - } - meta2 = get_datetime_metadata_from_dtype(type2); - if (meta2 == NULL) { - return NULL; - } - /* Take the maximum of the events */ if (meta1->events > meta2->events) { events = meta1->events; @@ -1770,50 +1759,94 @@ compute_datetime_metadata_greatest_common_divisor( /* Compute the GCD of the resulting multipliers */ num = _uint64_euclidean_gcd(num1, num2); - /* Create and return the metadata capsule */ - dt_data = PyArray_malloc(sizeof(PyArray_DatetimeMetaData)); - if (dt_data == NULL) { - return PyErr_NoMemory(); - } - - dt_data->base = base; - dt_data->num = (int)num; - if (dt_data->num <= 0 || num != (npy_uint64)dt_data->num) { + /* Fill the 'out_meta' values */ + out_meta->base = base; + out_meta->num = (int)num; + if (out_meta->num <= 0 || num != (npy_uint64)out_meta->num) { goto units_overflow; } - dt_data->events = events; + out_meta->events = events; - return NpyCapsule_FromVoidPtr((void *)dt_data, simple_capsule_dtor); + return 0; incompatible_units: { PyObject *errmsg; errmsg = PyUString_FromString("Cannot get " - "a common metadata divisor for types "); - PyUString_ConcatAndDel(&errmsg, - PyObject_Repr((PyObject *)type1)); + "a common metadata divisor for " + "NumPy datetime metadata "); + errmsg = append_metastr_to_string(meta1, 0, errmsg); PyUString_ConcatAndDel(&errmsg, PyUString_FromString(" and ")); - PyUString_ConcatAndDel(&errmsg, - PyObject_Repr((PyObject *)type2)); + errmsg = append_metastr_to_string(meta2, 0, errmsg); PyUString_ConcatAndDel(&errmsg, PyUString_FromString(" because they have " "incompatible nonlinear base time units")); PyErr_SetObject(PyExc_TypeError, errmsg); - return NULL; + return -1; } units_overflow: { PyObject *errmsg; errmsg = PyUString_FromString("Integer overflow " - "getting a common metadata divisor for types "); - PyUString_ConcatAndDel(&errmsg, - PyObject_Repr((PyObject *)type1)); + "getting a common metadata divisor for " + "NumPy datetime metadata "); + errmsg = append_metastr_to_string(meta1, 0, errmsg); PyUString_ConcatAndDel(&errmsg, PyUString_FromString(" and ")); - PyUString_ConcatAndDel(&errmsg, - PyObject_Repr((PyObject *)type2)); + errmsg = append_metastr_to_string(meta2, 0, errmsg); PyErr_SetObject(PyExc_OverflowError, errmsg); + return -1; + } +} + +/* + * Computes the GCD of the two date-time metadata values. Raises + * an exception if there is no reasonable GCD, such as with + * years and days. + * + * Returns a capsule with the GCD metadata. + */ +NPY_NO_EXPORT PyObject * +compute_datetime_metadata_greatest_common_divisor_capsule( + PyArray_Descr *type1, + PyArray_Descr *type2, + int strict_with_nonlinear_units1, + int strict_with_nonlinear_units2) +{ + PyArray_DatetimeMetaData *meta1, *meta2, *dt_data; + + if ((type1->type_num != NPY_DATETIME && + type1->type_num != NPY_TIMEDELTA) || + (type2->type_num != NPY_DATETIME && + type2->type_num != NPY_TIMEDELTA)) { + PyErr_SetString(PyExc_TypeError, + "Require datetime types for metadata " + "greatest common divisor operation"); + return NULL; + } + + meta1 = get_datetime_metadata_from_dtype(type1); + if (meta1 == NULL) { + return NULL; + } + meta2 = get_datetime_metadata_from_dtype(type2); + if (meta2 == NULL) { return NULL; } + + /* Create and return the metadata capsule */ + dt_data = PyArray_malloc(sizeof(PyArray_DatetimeMetaData)); + if (dt_data == NULL) { + return PyErr_NoMemory(); + } + + if (compute_datetime_metadata_greatest_common_divisor(meta1, meta2, + dt_data, strict_with_nonlinear_units1, + strict_with_nonlinear_units2) < 0) { + PyArray_free(dt_data); + return NULL; + } + + return NpyCapsule_FromVoidPtr((void *)dt_data, simple_capsule_dtor); } /* @@ -1838,7 +1871,7 @@ datetime_type_promotion(PyArray_Descr *type1, PyArray_Descr *type2) * Get the metadata GCD, being strict about nonlinear units for * timedelta and relaxed for datetime. */ - gcdmeta = compute_datetime_metadata_greatest_common_divisor( + gcdmeta = compute_datetime_metadata_greatest_common_divisor_capsule( type1, type2, type_num1 == NPY_TIMEDELTA, type_num2 == NPY_TIMEDELTA); @@ -2069,7 +2102,7 @@ convert_datetime_metadata_tuple_to_metacobj(PyObject *tuple) */ NPY_NO_EXPORT int convert_pyobject_to_datetime_metadata(PyObject *obj, - PyArray_DatetimeMetaData *out_meta) + PyArray_DatetimeMetaData *out_meta) { PyObject *ascii = NULL; char *str = NULL; @@ -3693,22 +3726,12 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, return 0; } /* Do no conversion on raw integers */ - else if (PyInt_Check(obj)) { - /* Use the default unit if none was specified */ - if (meta->base == -1) { - meta->base = NPY_DATETIME_DEFAULTUNIT; - meta->num = 1; - meta->events = 1; - } - *out = PyInt_AS_LONG(obj); - return 0; - } - else if (PyLong_Check(obj)) { - /* Use the default unit if none was specified */ + else if (PyInt_Check(obj) || PyLong_Check(obj)) { + /* Don't allow conversion from an integer without specifying a unit */ if (meta->base == -1) { - meta->base = NPY_DATETIME_DEFAULTUNIT; - meta->num = 1; - meta->events = 1; + PyErr_SetString(PyExc_ValueError, "Converting an integer to a " + "NumPy datetime requires a specified unit"); + return -1; } *out = PyLong_AsLongLong(obj); return 0; @@ -4306,7 +4329,7 @@ is_any_numpy_datetime(PyObject *obj) (PyArray_Check(obj) && ( PyArray_DESCR(obj)->type_num == NPY_DATETIME)) || PyDate_Check(obj) || - PyDatetime_Check(obj)); + PyDateTime_Check(obj)); } /* @@ -4338,7 +4361,14 @@ NPY_NO_EXPORT PyObject * datetime_arange(PyObject *start, PyObject *stop, PyObject *step, PyArray_Descr *dtype) { - PyArray_DatetimeMetaData meta_start, meta_stop, meta_step; + PyArray_DatetimeMetaData meta_value, meta_step; + int is_timedelta = 0; + /* + * Both datetime and timedelta are stored as int64, so they can + * share value variables. + */ + npy_int64 start_value = 0, stop_value = 0; + /* * First normalize the input parameters so there is no Py_None, * and start is moved to stop if stop is unspecified. @@ -4362,8 +4392,260 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, /* * Now figure out whether we're doing a datetime or a timedelta - * arange, and extract the values. + * arange, and get the metadata for start and stop. */ + if (dtype != NULL) { + PyArray_DatetimeMetaData *meta; + + if (dtype->type_num == NPY_TIMEDELTA) { + is_timedelta = 1; + } + else if (dtype->type_num != NPY_DATETIME) { + PyErr_SetString(PyExc_ValueError, + "datetime_arange was given a non-datetime dtype"); + return NULL; + } + + meta = get_datetime_metadata_from_dtype(dtype); + if (meta == NULL) { + return NULL; + } + + meta_value = *meta; + + if (is_timedelta) { + /* Get 'start', defaulting to 0 if not provided */ + if (start == NULL) { + start_value = 0; + } + else { + if (convert_pyobject_to_timedelta(&meta_value, start, + &start_value) < 0) { + return NULL; + } + } + /* Get 'stop' */ + if (convert_pyobject_to_timedelta(&meta_value, stop, + &stop_value) < 0) { + return NULL; + } + } + else { + /* Get 'start', raising an exception if it is not provided */ + if (start == NULL) { + PyErr_SetString(PyExc_ValueError, + "arange requires both a start and a stop for " + "NumPy datetime64 ranges"); + return NULL; + } + else { + if (convert_pyobject_to_datetime(&meta_value, start, + &start_value) < 0) { + return NULL; + } + } + /* + * Get 'stop', treating it specially if it's an integer + * or a timedelta. + */ + if (PyInt_Check(stop) || + PyLong_Check(stop) || + PyArray_IsScalar(stop, Integer) || + is_any_numpy_timedelta(stop)) { + if (convert_pyobject_to_timedelta(&meta_value, stop, + &stop_value) < 0) { + return NULL; + } + + /* Add the start value to make it a datetime */ + stop_value += start_value; + } + else { + if (convert_pyobject_to_datetime(&meta_value, stop, + &stop_value) < 0) { + return NULL; + } + } + } + } + /* if dtype is NULL */ + else { + /* Datetime arange */ + if (start != NULL && is_any_numpy_datetime(start)) { + PyArray_DatetimeMetaData meta_tmp1, meta_tmp2; + npy_datetime tmp1 = 0, tmp2 = 0; + + /* Get 'start' as a datetime */ + meta_tmp1.base = -1; + if (convert_pyobject_to_datetime(&meta_tmp1, start, + &tmp1) < 0) { + return NULL; + } + + /* + * If 'stop' is a timedelta, resolve the metadata + * and treat it specially + */ + if (is_any_numpy_timedelta(stop)) { + meta_tmp2.base = -1; + /* Convert to a timedelta */ + if (convert_pyobject_to_timedelta(&meta_tmp2, stop, + &tmp2) < 0) { + return NULL; + } + + /* Merge the metadata */ + if (compute_datetime_metadata_greatest_common_divisor( + &meta_tmp1, &meta_tmp2, + &meta_value, 1) < 0) { + return NULL; + } + + /* Convert 'start' to the merged metadata */ + if (cast_datetime_to_datetime(&meta_tmp1, &meta_value, + tmp1, &start_value) < 0) { + return NULL; + } + + /* Convert the timedelta to the merged metadata */ + if (cast_timedelta_to_timedelta(&meta_tmp2, &meta_value, + tmp2, &stop_value) < 0) { + return NULL; + } + + /* Add the start to the stop timedelta */ + stop_value += start_value; + } + /* + * If 'stop' is an integer, treat it specially + * like a timedelta with units matching the start datetime + */ + else if (PyInt_Check(stop) || + PyLong_Check(stop) || + PyArray_IsScalar(stop, Integer)) { + npy_longlong tmp; + + /* Get the integer value */ + tmp = PyLong_AsLongLong(stop); + if (tmp == -1 && PyErr_Occurred()) { + return NULL; + } + + /* Copy the 'start' metadata and value */ + meta_value = meta_tmp1; + start_value = tmp1; + + stop_value = start_value + tmp; + } + /* Otherwise try to interpret 'stop' as a datetime */ + else { + meta_tmp2.base = -1; + /* Convert to a datetime */ + if (convert_pyobject_to_datetime(&meta_tmp2, stop, + &tmp2) < 0) { + return NULL; + } + + /* Merge the metadata */ + if (compute_datetime_metadata_greatest_common_divisor( + &meta_tmp1, &meta_tmp2, + &meta_value, 0) < 0) { + return NULL; + } + + /* Convert 'start' to the merged metadata */ + if (cast_datetime_to_datetime(&meta_tmp1, &meta_value, + tmp1, &start_value) < 0) { + return NULL; + } + + /* Convert 'stop' to the merged metadata */ + if (cast_datetime_to_datetime(&meta_tmp2, &meta_value, + tmp2, &stop_value) < 0) { + return NULL; + } + } + } + /* Timedelta arange */ + else { + PyArray_DatetimeMetaData meta_tmp1, meta_tmp2; + npy_timedelta tmp1 = 0, tmp2 = 0; + + is_timedelta = 1; + + if (start == NULL || + PyInt_Check(start) || + PyLong_Check(start) || + PyArray_IsScalar(start, Integer)) { + if (start == NULL) { + start_value = 0; + } + else { + start_value = PyLong_AsLongLong(start); + if (start_value == -1 && PyErr_Occurred()) { + return NULL; + } + } + + meta_value.base = -1; + /* Convert 'stop' to a timedelta */ + if (convert_pyobject_to_timedelta(&meta_value, stop, + &stop_value) < 0) { + return NULL; + } + } + else { + meta_tmp1.base = -1; + /* Convert 'start' to a timedelta */ + if (convert_pyobject_to_timedelta(&meta_tmp1, start, + &tmp1) < 0) { + return NULL; + } + + if (PyInt_Check(stop) || + PyLong_Check(stop) || + PyArray_IsScalar(stop, Integer)) { + /* Use the metadata and value from 'start' */ + meta_value = meta_tmp1; + start_value = tmp1; + + /* Get the integer value for stop */ + stop_value = PyLong_AsLongLong(stop); + if (stop_value == -1 && PyErr_Occurred()) { + return NULL; + } + } + /* Otherwise try to interpret 'stop' as a timedelta */ + else { + meta_tmp2.base = -1; + /* Convert to a datetime */ + if (convert_pyobject_to_timedelta(&meta_tmp2, stop, + &tmp2) < 0) { + return NULL; + } + + /* Merge the metadata */ + if (compute_datetime_metadata_greatest_common_divisor( + &meta_tmp1, &meta_tmp2, + &meta_value, 1) < 0) { + return NULL; + } + + /* Convert 'start' to the merged metadata */ + if (cast_timedelta_to_timedelta(&meta_tmp1, &meta_value, + tmp1, &start_value) < 0) { + return NULL; + } + + /* Convert 'stop' to the merged metadata */ + if (cast_timedelta_to_timedelta(&meta_tmp2, &meta_value, + tmp2, &stop_value) < 0) { + return NULL; + } + } + } + } + } PyErr_SetString(PyExc_RuntimeError, "datetime_arange is incomplete"); return NULL; From d6c63e31c8761bff897b4bdf8146aa5704ada0f9 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Wed, 8 Jun 2011 17:56:25 -0500 Subject: [PATCH 07/34] ENH: datetime-meta: Add generic units as a datetime unit type This allows integers to convert into timedeltas without binding to a default unit, so that later when it's combined with another data type it adopts that type instead of overriding it haphazardly. This makes things generally more intuitive. --- numpy/core/include/numpy/ndarraytypes.h | 7 +- numpy/core/src/multiarray/datetime.c | 166 +++++++++++++++----- numpy/core/src/multiarray/dtype_transfer.c | 10 -- numpy/core/src/multiarray/scalartypes.c.src | 9 +- numpy/core/tests/test_datetime.py | 133 +++++++++------- 5 files changed, 215 insertions(+), 110 deletions(-) diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 5bed8d70cd49..7d04c0106ba9 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -243,11 +243,12 @@ typedef enum { NPY_FR_ns,/* nanoseconds */ NPY_FR_ps,/* picoseconds */ NPY_FR_fs,/* femtoseconds */ - NPY_FR_as /* attoseconds */ + NPY_FR_as,/* attoseconds */ + NPY_FR_GENERIC /* Generic, unbound units, can convert to anything */ } NPY_DATETIMEUNIT; -#define NPY_DATETIME_NUMUNITS (NPY_FR_as + 1) -#define NPY_DATETIME_DEFAULTUNIT NPY_FR_us +#define NPY_DATETIME_NUMUNITS (NPY_FR_GENERIC + 1) +#define NPY_DATETIME_DEFAULTUNIT NPY_FR_GENERIC #define NPY_STR_Y "Y" #define NPY_STR_M "M" diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index a8970dc7bab2..cb28906c1fbb 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -27,10 +27,6 @@ numpy_pydatetime_import() static int is_leapyear(npy_int64 year); - -/* For defaults and errors */ -#define NPY_FR_ERR -1 - /* Exported as DATETIMEUNITS in multiarraymodule.c */ NPY_NO_EXPORT char *_datetime_strings[] = { NPY_STR_Y, @@ -46,7 +42,8 @@ NPY_NO_EXPORT char *_datetime_strings[] = { NPY_STR_ns, NPY_STR_ps, NPY_STR_fs, - NPY_STR_as + NPY_STR_as, + "generic" }; /* @@ -251,6 +248,14 @@ convert_datetimestruct_to_datetime(PyArray_DatetimeMetaData *meta, return 0; } + /* Cannot instantiate a datetime with generic units */ + if (meta->base == NPY_FR_GENERIC) { + PyErr_SetString(PyExc_ValueError, + "Cannot create a NumPy datetime other than NaT " + "with generic units"); + return -1; + } + if (dts->event < 0 || dts->event >= meta->events) { PyErr_Format(PyExc_ValueError, "NumPy datetime event %d is outside range [0,%d)", @@ -545,6 +550,14 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, out->year = NPY_DATETIME_NAT; return 0; } + + /* Datetimes can't be in generic units */ + if (meta->base == NPY_FR_GENERIC) { + PyErr_SetString(PyExc_ValueError, + "Cannot convert a NumPy datetime value other than NaT " + "with generic units"); + return -1; + } /* Extract the event number */ if (meta->events > 1) { @@ -1101,6 +1114,15 @@ parse_datetime_metadata_from_metastr(char *metastr, Py_ssize_t len, { char *substr = metastr, *substrend = NULL; + /* Treat the empty string as generic units */ + if (len == 0) { + out_meta->base = NPY_FR_GENERIC; + out_meta->num = 1; + out_meta->events = 1; + + return 0; + } + /* The metadata string must start with a '[' */ if (len < 3 || *substr++ != '[') { goto bad_input; @@ -1313,6 +1335,12 @@ convert_datetime_divisor_to_multiple(PyArray_DatetimeMetaData *meta, NPY_DATETIMEUNIT *baseunit; int q, r; + if (meta->base == NPY_FR_GENERIC) { + PyErr_SetString(PyExc_ValueError, + "Can't use 'den' divisor with generic units"); + return -1; + } + ind = ((int)meta->base - (int)NPY_FR_Y)*2; totry = _multiples_table[ind]; baseunit = _multiples_table[ind + 1]; @@ -1330,10 +1358,10 @@ convert_datetime_divisor_to_multiple(PyArray_DatetimeMetaData *meta, baseunit = _multiples_table[ind + 1]; baseunit[0] = meta->base + 1; baseunit[1] = meta->base + 2; - if (meta->base == NPY_DATETIME_NUMUNITS - 2) { + if (meta->base == NPY_FR_as - 1) { num = 1; } - if (meta->base == NPY_DATETIME_NUMUNITS - 1) { + if (meta->base == NPY_FR_as) { num = 0; } } @@ -1383,12 +1411,14 @@ _datetime_factors[] = { 1000, 1000, 1000, - 1 /* Attoseconds are the smallest base unit */ + 1, /* Attoseconds are the smallest base unit */ + 0 /* Generic units don't have a conversion */ }; /* * Returns the scale factor between the units. Does not validate - * that bigbase represents larger units than littlebase. + * that bigbase represents larger units than littlebase, or that + * the units are not generic. * * Returns 0 if there is an overflow. */ @@ -1451,6 +1481,25 @@ get_datetime_conversion_factor(PyArray_DatetimeMetaData *src_meta, int src_base, dst_base, swapped; npy_uint64 num = 1, denom = 1, tmp, gcd; + /* Generic units change to the destination with no conversion factor */ + if (src_meta->base == NPY_FR_GENERIC) { + *out_num = 1; + *out_denom = 1; + return; + } + /* + * Converting to a generic unit from something other than a generic + * unit is an error. + */ + else if (dst_meta->base == NPY_FR_GENERIC) { + PyErr_SetString(PyExc_ValueError, + "Cannot convert from specific units to generic " + "units in NumPy datetimes or timedeltas"); + *out_num = 0; + *out_denom = 0; + return; + } + if (src_meta->base <= dst_meta->base) { src_base = src_meta->base; dst_base = dst_meta->base; @@ -1503,6 +1552,11 @@ get_datetime_conversion_factor(PyArray_DatetimeMetaData *src_meta, /* If something overflowed, make both num and denom 0 */ if (denom == 0) { + PyErr_Format(PyExc_OverflowError, + "Integer overflow getting a conversion factor between " + "NumPy datetime unit %s and %s", + _datetime_strings[src_base], + _datetime_strings[dst_base]); *out_num = 0; *out_denom = 0; return; @@ -1556,6 +1610,15 @@ datetime_metadata_divides( return 0; } + /* Generic units divide into anything */ + if (meta2->base == NPY_FR_GENERIC) { + return 1; + } + /* Non-generic units never divide into generic units */ + else if (meta1->base == NPY_FR_GENERIC) { + return 0; + } + /* Events must match */ if (meta1->events != meta2->events) { return 0; @@ -1652,6 +1715,16 @@ compute_datetime_metadata_greatest_common_divisor( npy_uint64 num1, num2, num; int events = 1; + /* If either unit is generic, adopt the metadata from the other one */ + if (meta1->base == NPY_FR_GENERIC) { + *out_meta = *meta2; + return 0; + } + else if (meta2->base == NPY_FR_GENERIC) { + *out_meta = *meta1; + return 0; + } + /* Take the maximum of the events */ if (meta1->events > meta2->events) { events = meta1->events; @@ -1918,6 +1991,8 @@ datetime_type_promotion(PyArray_Descr *type1, PyArray_Descr *type2) * a date time unit enum value. The 'metastr' parameter * is used for error messages, and may be NULL. * + * Generic units have no representation as a string in this form. + * * Returns 0 on success, -1 on failure. */ NPY_NO_EXPORT NPY_DATETIMEUNIT @@ -2173,6 +2248,18 @@ append_metastr_to_string(PyArray_DatetimeMetaData *meta, return NULL; } + if (meta->base == NPY_FR_GENERIC) { + /* Without brackets, give a string "generic" */ + if (skip_brackets) { + PyUString_ConcatAndDel(&ret, PyUString_FromString("generic")); + return ret; + } + /* But with brackets, append nothing */ + else { + return ret; + } + } + num = meta->num; events = meta->events; if (meta->base >= 0 && meta->base < NPY_DATETIME_NUMUNITS) { @@ -2349,14 +2436,13 @@ parse_iso_8601_date(char *str, int len, /* * Indicate that this was a special value, and - * recommend 'Y' as the unit because when promoted - * with any other unit, will produce that unit. + * recommend generic units. */ if (out_local != NULL) { *out_local = 0; } if (out_bestunit != NULL) { - *out_bestunit = NPY_FR_Y; + *out_bestunit = NPY_FR_GENERIC; } if (out_special != NULL) { *out_special = 1; @@ -2365,6 +2451,13 @@ parse_iso_8601_date(char *str, int len, return 0; } + if (unit == NPY_FR_GENERIC) { + PyErr_SetString(PyExc_ValueError, + "Cannot create a NumPy datetime other than NaT " + "with generic units"); + return -1; + } + /* * The string "today" resolves to midnight of today's local date in UTC. * This is perhaps a little weird, but done so that further truncation @@ -2943,6 +3036,9 @@ get_datetime_iso_8601_strlen(int local, NPY_DATETIMEUNIT base) } switch (base) { + /* Generic units can only be used to represent NaT */ + case NPY_FR_GENERIC: + return 4; case NPY_FR_as: len += 3; /* "###" */ case NPY_FR_fs: @@ -3013,8 +3109,8 @@ make_iso_8601_date(npy_datetimestruct *dts, char *outstr, int outlen, char *substr = outstr, sublen = outlen; int tmplen; - /* Handle NaT */ - if (dts->year == NPY_DATETIME_NAT) { + /* Handle NaT, and treat a datetime with generic units as NaT */ + if (dts->year == NPY_DATETIME_NAT || base == NPY_FR_GENERIC) { if (outlen < 4) { goto string_too_short; } @@ -3728,7 +3824,7 @@ convert_pyobject_to_datetime(PyArray_DatetimeMetaData *meta, PyObject *obj, /* Do no conversion on raw integers */ else if (PyInt_Check(obj) || PyLong_Check(obj)) { /* Don't allow conversion from an integer without specifying a unit */ - if (meta->base == -1) { + if (meta->base == -1 || meta->base == NPY_FR_GENERIC) { PyErr_SetString(PyExc_ValueError, "Converting an integer to a " "NumPy datetime requires a specified unit"); return -1; @@ -3851,18 +3947,7 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj, npy_timedelta *out) { /* Do no conversion on raw integers */ - if (PyInt_Check(obj)) { - /* Use the default unit if none was specified */ - if (meta->base == -1) { - meta->base = NPY_DATETIME_DEFAULTUNIT; - meta->num = 1; - meta->events = 1; - } - - *out = PyInt_AS_LONG(obj); - return 0; - } - else if (PyLong_Check(obj)) { + if (PyInt_Check(obj) || PyLong_Check(obj)) { /* Use the default unit if none was specified */ if (meta->base == -1) { meta->base = NPY_DATETIME_DEFAULTUNIT; @@ -4008,8 +4093,8 @@ convert_datetime_to_pyobject(npy_datetime dt, PyArray_DatetimeMetaData *meta) PyObject *ret = NULL, *tup = NULL; npy_datetimestruct dts; - /* Handle not-a-time */ - if (dt == NPY_DATETIME_NAT) { + /* Handle not-a-time, and generic units as NaT as well */ + if (dt == NPY_DATETIME_NAT || meta->base == NPY_FR_GENERIC) { return PyUString_FromString("NaT"); } @@ -4089,13 +4174,14 @@ convert_timedelta_to_pyobject(npy_timedelta td, PyArray_DatetimeMetaData *meta) } /* - * If the type's precision is greater than microseconds or is - * Y/M/B (nonlinear units), return an int + * If the type's precision is greater than microseconds, is + * Y/M/B (nonlinear units), or is generic units, return an int */ if (meta->base > NPY_FR_us || meta->base == NPY_FR_Y || meta->base == NPY_FR_M || - meta->base == NPY_FR_B) { + meta->base == NPY_FR_B || + meta->base == NPY_FR_GENERIC) { /* Skip use of a tuple for the events, just return the raw int */ return PyLong_FromLongLong(td); } @@ -4215,6 +4301,11 @@ has_equivalent_datetime_metadata(PyArray_Descr *type1, PyArray_Descr *type2) return 0; } + /* For generic units, the num and events are ignored */ + if (meta1->base == NPY_FR_GENERIC && meta2->base == NPY_FR_GENERIC) { + return 1; + } + return meta1->base == meta2->base && meta1->num == meta2->num && meta1->events == meta2->events; @@ -4285,9 +4376,6 @@ cast_timedelta_to_timedelta(PyArray_DatetimeMetaData *src_meta, get_datetime_conversion_factor(src_meta, dst_meta, &num, &denom); if (num == 0) { - PyErr_SetString(PyExc_OverflowError, - "Integer overflow getting a conversion factor between " - "different timedelta types"); return -1; } @@ -4361,7 +4449,7 @@ NPY_NO_EXPORT PyObject * datetime_arange(PyObject *start, PyObject *stop, PyObject *step, PyArray_Descr *dtype) { - PyArray_DatetimeMetaData meta_value, meta_step; + PyArray_DatetimeMetaData meta_value; int is_timedelta = 0; /* * Both datetime and timedelta are stored as int64, so they can @@ -4497,7 +4585,7 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, /* Merge the metadata */ if (compute_datetime_metadata_greatest_common_divisor( &meta_tmp1, &meta_tmp2, - &meta_value, 1) < 0) { + &meta_value, 0, 1) < 0) { return NULL; } @@ -4549,7 +4637,7 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, /* Merge the metadata */ if (compute_datetime_metadata_greatest_common_divisor( &meta_tmp1, &meta_tmp2, - &meta_value, 0) < 0) { + &meta_value, 0, 0) < 0) { return NULL; } @@ -4627,7 +4715,7 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, /* Merge the metadata */ if (compute_datetime_metadata_greatest_common_divisor( &meta_tmp1, &meta_tmp2, - &meta_value, 1) < 0) { + &meta_value, 1, 1) < 0) { return NULL; } diff --git a/numpy/core/src/multiarray/dtype_transfer.c b/numpy/core/src/multiarray/dtype_transfer.c index 14a6c9d6a9a9..8d53a4b653b0 100644 --- a/numpy/core/src/multiarray/dtype_transfer.c +++ b/numpy/core/src/multiarray/dtype_transfer.c @@ -868,16 +868,6 @@ get_nbo_cast_datetime_transfer_function(int aligned, get_datetime_conversion_factor(src_meta, dst_meta, &num, &denom); if (num == 0) { - PyObject *errmsg; - errmsg = PyUString_FromString("Integer overflow " - "getting a conversion factor between types "); - PyUString_ConcatAndDel(&errmsg, - PyObject_Repr((PyObject *)src_dtype)); - PyUString_ConcatAndDel(&errmsg, - PyUString_FromString(" and ")); - PyUString_ConcatAndDel(&errmsg, - PyObject_Repr((PyObject *)dst_dtype)); - PyErr_SetObject(PyExc_OverflowError, errmsg); return NPY_FAIL; } diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 50e51e5918c9..49ac76441444 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -701,7 +701,8 @@ static char *_datetime_verbose_strings[] = { "nanoseconds", "picoseconds", "femtoseconds", - "attoseconds" + "attoseconds", + "generic time units" }; static PyObject * @@ -2492,6 +2493,7 @@ finish: * #name = datetime, timedelta# * #Name = Datetime, Timedelta# * #NAME = DATETIME, TIMEDELTA# + * #is_datetime = 1, 0# */ static PyObject * @@ -2535,7 +2537,12 @@ static PyObject * ret->obmeta.events = 1; } + /* Make datetime default to NaT, timedelta default to zero */ +#if @is_datetime@ + ret->obval = NPY_DATETIME_NAT; +#else ret->obval = 0; +#endif } else if (convert_pyobject_to_@name@(&ret->obmeta, obj, &ret->obval) < 0) { diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index 994066137a91..2d13d399e579 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -14,7 +14,10 @@ def test_datetime_dtype_creation(self): assert_(dt1 == np.dtype('datetime64[750%s]' % unit)) dt2 = np.dtype('m8[%s]' % unit) assert_(dt2 == np.dtype('timedelta64[%s]' % unit)) - + + # Generic units shouldn't add [] to the end + assert_equal(str(np.dtype("M8")), "datetime64") + # Check that the parser rejects bad datetime types assert_raises(TypeError, np.dtype, 'M8[badunit]') assert_raises(TypeError, np.dtype, 'm8[badunit]') @@ -80,8 +83,15 @@ def test_datetime_scalar_construction(self): assert_equal(np.datetime64('1950-03-12', 's'), np.datetime64('1950-03-12', 'm')) - # Default construction means 0 - assert_equal(np.datetime64(), np.datetime64(0)) + # Default construction means NaT + assert_equal(np.datetime64(), np.datetime64('NaT')) + + # Default construction of NaT is in generic units + assert_equal(np.datetime64().dtype, np.dtype('M8')) + assert_equal(np.datetime64('NaT').dtype, np.dtype('M8')) + + # Construction from integers requires a specified unit + assert_raises(ValueError, np.datetime64, 17) # When constructing from a scalar or zero-dimensional array, # it either keeps the units or you can override them. @@ -123,6 +133,9 @@ def test_timedelta_scalar_construction(self): # Default construction means 0 assert_equal(np.timedelta64(), np.timedelta64(0)) + # Construction from an integer produces generic units + assert_equal(np.timedelta64(12).dtype, np.dtype('m8')) + # When constructing from a scalar or zero-dimensional array, # it either keeps the units or you can override them. a = np.timedelta64(2, 'h') @@ -858,96 +871,102 @@ def test_divisor_conversion_as(self): def test_string_parser_variants(self): # Allow space instead of 'T' between date and time - assert_equal(np.array(['1980-02-29T01:02:03'], np.dtype('M8')), - np.array(['1980-02-29 01:02:03'], np.dtype('M8'))) + assert_equal(np.array(['1980-02-29T01:02:03'], np.dtype('M8[s]')), + np.array(['1980-02-29 01:02:03'], np.dtype('M8[s]'))) # Allow negative years - assert_equal(np.array(['-1980-02-29T01:02:03'], np.dtype('M8')), - np.array(['-1980-02-29 01:02:03'], np.dtype('M8'))) + assert_equal(np.array(['-1980-02-29T01:02:03'], np.dtype('M8[s]')), + np.array(['-1980-02-29 01:02:03'], np.dtype('M8[s]'))) # UTC specifier - assert_equal(np.array(['-1980-02-29T01:02:03Z'], np.dtype('M8')), - np.array(['-1980-02-29 01:02:03Z'], np.dtype('M8'))) + assert_equal(np.array(['-1980-02-29T01:02:03Z'], np.dtype('M8[s]')), + np.array(['-1980-02-29 01:02:03Z'], np.dtype('M8[s]'))) # Time zone offset - assert_equal(np.array(['1980-02-29T02:02:03Z'], np.dtype('M8')), - np.array(['1980-02-29 00:32:03-0130'], np.dtype('M8'))) - assert_equal(np.array(['1980-02-28T22:32:03Z'], np.dtype('M8')), - np.array(['1980-02-29 00:02:03+01:30'], np.dtype('M8'))) - assert_equal(np.array(['1980-02-29T02:32:03.506Z'], np.dtype('M8')), - np.array(['1980-02-29 00:32:03.506-02'], np.dtype('M8'))) + assert_equal(np.array(['1980-02-29T02:02:03Z'], np.dtype('M8[s]')), + np.array(['1980-02-29 00:32:03-0130'], np.dtype('M8[s]'))) + assert_equal(np.array(['1980-02-28T22:32:03Z'], np.dtype('M8[s]')), + np.array(['1980-02-29 00:02:03+01:30'], np.dtype('M8[s]'))) + assert_equal(np.array(['1980-02-29T02:32:03.506Z'], np.dtype('M8[s]')), + np.array(['1980-02-29 00:32:03.506-02'], np.dtype('M8[s]'))) assert_equal(np.datetime64('1977-03-02T12:30-0230'), np.datetime64('1977-03-02T15:00Z')) def test_string_parser_error_check(self): # Arbitrary bad string - assert_raises(ValueError, np.array, ['badvalue'], np.dtype('M8')) + assert_raises(ValueError, np.array, ['badvalue'], np.dtype('M8[us]')) # Character after year must be '-' - assert_raises(ValueError, np.array, ['1980X'], np.dtype('M8')) + assert_raises(ValueError, np.array, ['1980X'], np.dtype('M8[us]')) # Cannot have trailing '-' - assert_raises(ValueError, np.array, ['1980-'], np.dtype('M8')) + assert_raises(ValueError, np.array, ['1980-'], np.dtype('M8[us]')) # Month must be in range [1,12] - assert_raises(ValueError, np.array, ['1980-00'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-13'], np.dtype('M8')) + assert_raises(ValueError, np.array, ['1980-00'], np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-13'], np.dtype('M8[us]')) # Month must have two digits - assert_raises(ValueError, np.array, ['1980-1'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-1-02'], np.dtype('M8')) + assert_raises(ValueError, np.array, ['1980-1'], np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-1-02'], np.dtype('M8[us]')) # 'Mor' is not a valid month - assert_raises(ValueError, np.array, ['1980-Mor'], np.dtype('M8')) + assert_raises(ValueError, np.array, ['1980-Mor'], np.dtype('M8[us]')) # Cannot have trailing '-' - assert_raises(ValueError, np.array, ['1980-01-'], np.dtype('M8')) + assert_raises(ValueError, np.array, ['1980-01-'], np.dtype('M8[us]')) # Day must be in range [1,len(month)] - assert_raises(ValueError, np.array, ['1980-01-0'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-01-00'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-01-32'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1979-02-29'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-02-30'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-03-32'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-04-31'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-05-32'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-06-31'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-07-32'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-08-32'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-09-31'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-10-32'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-11-31'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-12-32'], np.dtype('M8')) + assert_raises(ValueError, np.array, ['1980-01-0'], np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-01-00'], np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-01-32'], np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1979-02-29'], np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-02-30'], np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-03-32'], np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-04-31'], np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-05-32'], np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-06-31'], np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-07-32'], np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-08-32'], np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-09-31'], np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-10-32'], np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-11-31'], np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-12-32'], np.dtype('M8[us]')) # Cannot have trailing characters - assert_raises(ValueError, np.array, ['1980-02-03%'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-02-03 q'], np.dtype('M8')) + assert_raises(ValueError, np.array, ['1980-02-03%'], + np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-02-03 q'], + np.dtype('M8[us]')) # Hours must be in range [0, 23] - assert_raises(ValueError, np.array, ['1980-02-03 25'], np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-02-03T25'], np.dtype('M8')) + assert_raises(ValueError, np.array, ['1980-02-03 25'], + np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-02-03T25'], + np.dtype('M8[us]')) assert_raises(ValueError, np.array, ['1980-02-03 24:01'], - np.dtype('M8')) + np.dtype('M8[us]')) assert_raises(ValueError, np.array, ['1980-02-03T24:01'], - np.dtype('M8')) - assert_raises(ValueError, np.array, ['1980-02-03 -1'], np.dtype('M8')) + np.dtype('M8[us]')) + assert_raises(ValueError, np.array, ['1980-02-03 -1'], + np.dtype('M8[us]')) # No trailing ':' - assert_raises(ValueError, np.array, ['1980-02-03 01:'], np.dtype('M8')) + assert_raises(ValueError, np.array, ['1980-02-03 01:'], + np.dtype('M8[us]')) # Minutes must be in range [0, 59] assert_raises(ValueError, np.array, ['1980-02-03 01:-1'], - np.dtype('M8')) + np.dtype('M8[us]')) assert_raises(ValueError, np.array, ['1980-02-03 01:60'], - np.dtype('M8')) + np.dtype('M8[us]')) # No trailing ':' assert_raises(ValueError, np.array, ['1980-02-03 01:60:'], - np.dtype('M8')) + np.dtype('M8[us]')) # Seconds must be in range [0, 59] assert_raises(ValueError, np.array, ['1980-02-03 01:10:-1'], - np.dtype('M8')) + np.dtype('M8[us]')) assert_raises(ValueError, np.array, ['1980-02-03 01:01:60'], - np.dtype('M8')) + np.dtype('M8[us]')) # Timezone offset must within a reasonable range assert_raises(ValueError, np.array, ['1980-02-03 01:01:00+0661'], - np.dtype('M8')) + np.dtype('M8[us]')) assert_raises(ValueError, np.array, ['1980-02-03 01:01:00+2500'], - np.dtype('M8')) + np.dtype('M8[us]')) assert_raises(ValueError, np.array, ['1980-02-03 01:01:00-0070'], - np.dtype('M8')) + np.dtype('M8[us]')) assert_raises(ValueError, np.array, ['1980-02-03 01:01:00-3000'], - np.dtype('M8')) + np.dtype('M8[us]')) assert_raises(ValueError, np.array, ['1980-02-03 01:01:00-25:00'], - np.dtype('M8')) + np.dtype('M8[us]')) def test_creation_overflow(self): From 5c164119cb133c181890e04329bdadc4a69cd05d Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Thu, 9 Jun 2011 09:50:40 -0500 Subject: [PATCH 08/34] ENH: datetime-arange: Use the generic units for parameter conversion --- numpy/core/src/multiarray/datetime.c | 291 ++++++++++++++------------- 1 file changed, 146 insertions(+), 145 deletions(-) diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index cb28906c1fbb..3a208bb3b850 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -4449,13 +4449,13 @@ NPY_NO_EXPORT PyObject * datetime_arange(PyObject *start, PyObject *stop, PyObject *step, PyArray_Descr *dtype) { - PyArray_DatetimeMetaData meta_value; + PyArray_DatetimeMetaData meta; int is_timedelta = 0; /* * Both datetime and timedelta are stored as int64, so they can * share value variables. */ - npy_int64 start_value = 0, stop_value = 0; + npy_int64 start_value = 0, stop_value = 0, step_value; /* * First normalize the input parameters so there is no Py_None, @@ -4478,12 +4478,19 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, start = NULL; } + /* Step must not be a Datetime */ + if (step != NULL && is_any_numpy_datetime(step)) { + PyErr_SetString(PyExc_ValueError, + "cannot use a datetime as a step in arange"); + return NULL; + } + /* * Now figure out whether we're doing a datetime or a timedelta * arange, and get the metadata for start and stop. */ if (dtype != NULL) { - PyArray_DatetimeMetaData *meta; + PyArray_DatetimeMetaData *meta_tmp; if (dtype->type_num == NPY_TIMEDELTA) { is_timedelta = 1; @@ -4494,12 +4501,12 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, return NULL; } - meta = get_datetime_metadata_from_dtype(dtype); - if (meta == NULL) { + meta_tmp = get_datetime_metadata_from_dtype(dtype); + if (meta_tmp == NULL) { return NULL; } - meta_value = *meta; + meta = *meta_tmp; if (is_timedelta) { /* Get 'start', defaulting to 0 if not provided */ @@ -4507,13 +4514,13 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, start_value = 0; } else { - if (convert_pyobject_to_timedelta(&meta_value, start, + if (convert_pyobject_to_timedelta(&meta, start, &start_value) < 0) { return NULL; } } /* Get 'stop' */ - if (convert_pyobject_to_timedelta(&meta_value, stop, + if (convert_pyobject_to_timedelta(&meta, stop, &stop_value) < 0) { return NULL; } @@ -4527,7 +4534,7 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, return NULL; } else { - if (convert_pyobject_to_datetime(&meta_value, start, + if (convert_pyobject_to_datetime(&meta, start, &start_value) < 0) { return NULL; } @@ -4540,7 +4547,7 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, PyLong_Check(stop) || PyArray_IsScalar(stop, Integer) || is_any_numpy_timedelta(stop)) { - if (convert_pyobject_to_timedelta(&meta_value, stop, + if (convert_pyobject_to_timedelta(&meta, stop, &stop_value) < 0) { return NULL; } @@ -4549,7 +4556,7 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, stop_value += start_value; } else { - if (convert_pyobject_to_datetime(&meta_value, stop, + if (convert_pyobject_to_datetime(&meta, stop, &stop_value) < 0) { return NULL; } @@ -4558,179 +4565,173 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, } /* if dtype is NULL */ else { - /* Datetime arange */ - if (start != NULL && is_any_numpy_datetime(start)) { - PyArray_DatetimeMetaData meta_tmp1, meta_tmp2; - npy_datetime tmp1 = 0, tmp2 = 0; - - /* Get 'start' as a datetime */ - meta_tmp1.base = -1; - if (convert_pyobject_to_datetime(&meta_tmp1, start, - &tmp1) < 0) { - return NULL; - } + /* + * Start meta as generic units, so taking the gcd + * does the right thing + */ + meta.base = NPY_FR_GENERIC; - /* - * If 'stop' is a timedelta, resolve the metadata - * and treat it specially - */ - if (is_any_numpy_timedelta(stop)) { - meta_tmp2.base = -1; - /* Convert to a timedelta */ - if (convert_pyobject_to_timedelta(&meta_tmp2, stop, - &tmp2) < 0) { - return NULL; - } + /* Timedelta arange */ + if (!is_any_numpy_datetime(start) && !is_any_numpy_datetime(stop)) { + PyArray_DatetimeMetaData meta_tmp, meta_mrg; - /* Merge the metadata */ - if (compute_datetime_metadata_greatest_common_divisor( - &meta_tmp1, &meta_tmp2, - &meta_value, 0, 1) < 0) { - return NULL; - } + is_timedelta = 1; - /* Convert 'start' to the merged metadata */ - if (cast_datetime_to_datetime(&meta_tmp1, &meta_value, - tmp1, &start_value) < 0) { + /* Convert start to a timedelta */ + if (start == NULL) { + start_value = 0; + } + else { + /* Directly copy to meta, no point in gcd for 'start' */ + meta.base = -1; + /* Convert 'start' to a timedelta */ + if (convert_pyobject_to_timedelta(&meta, start, + &start_value) < 0) { return NULL; } + } - /* Convert the timedelta to the merged metadata */ - if (cast_timedelta_to_timedelta(&meta_tmp2, &meta_value, - tmp2, &stop_value) < 0) { - return NULL; - } + /* Temporary metadata, to merge with 'meta' */ + meta_tmp.base = -1; + /* Convert 'stop' to a timedelta */ + if (convert_pyobject_to_timedelta(&meta_tmp, stop, + &stop_value) < 0) { + return NULL; + } - /* Add the start to the stop timedelta */ - stop_value += start_value; + /* Merge the metadata */ + if (compute_datetime_metadata_greatest_common_divisor( + &meta, &meta_tmp, &meta_mrg, 1, 1) < 0) { + return NULL; } - /* - * If 'stop' is an integer, treat it specially - * like a timedelta with units matching the start datetime - */ - else if (PyInt_Check(stop) || - PyLong_Check(stop) || - PyArray_IsScalar(stop, Integer)) { - npy_longlong tmp; - /* Get the integer value */ - tmp = PyLong_AsLongLong(stop); - if (tmp == -1 && PyErr_Occurred()) { - return NULL; - } + /* Cast 'start' and 'stop' to the merged metadata */ + if (cast_timedelta_to_timedelta(&meta, &meta_mrg, + start_value, &start_value) < 0) { + return NULL; + } + if (cast_timedelta_to_timedelta(&meta_tmp, &meta_mrg, + stop_value, &stop_value) < 0) { + return NULL; + } - /* Copy the 'start' metadata and value */ - meta_value = meta_tmp1; - start_value = tmp1; + meta = meta_mrg; - stop_value = start_value + tmp; + if (step == NULL) { + step_value = 1; } - /* Otherwise try to interpret 'stop' as a datetime */ else { - meta_tmp2.base = -1; - /* Convert to a datetime */ - if (convert_pyobject_to_datetime(&meta_tmp2, stop, - &tmp2) < 0) { + /* Temporary metadata, to merge with 'meta' */ + meta_tmp.base = -1; + /* Convert 'step' to a timedelta */ + if (convert_pyobject_to_timedelta(&meta_tmp, step, + &step_value) < 0) { return NULL; } /* Merge the metadata */ if (compute_datetime_metadata_greatest_common_divisor( - &meta_tmp1, &meta_tmp2, - &meta_value, 0, 0) < 0) { + &meta, &meta_tmp, &meta_mrg, 1, 1) < 0) { return NULL; } - /* Convert 'start' to the merged metadata */ - if (cast_datetime_to_datetime(&meta_tmp1, &meta_value, - tmp1, &start_value) < 0) { + /* Cast 'start', 'stop', and 'step' to the merged metadata */ + if (cast_timedelta_to_timedelta(&meta, &meta_mrg, + start_value, &start_value) < 0) { return NULL; } - - /* Convert 'stop' to the merged metadata */ - if (cast_datetime_to_datetime(&meta_tmp2, &meta_value, - tmp2, &stop_value) < 0) { + if (cast_timedelta_to_timedelta(&meta, &meta_mrg, + stop_value, &stop_value) < 0) { return NULL; } + if (cast_timedelta_to_timedelta(&meta_tmp, &meta_mrg, + step_value, &step_value) < 0) { + return NULL; + } + + meta = meta_mrg; } } - /* Timedelta arange */ + /* Datetime arange */ else { - PyArray_DatetimeMetaData meta_tmp1, meta_tmp2; - npy_timedelta tmp1 = 0, tmp2 = 0; - - is_timedelta = 1; + PyArray_DatetimeMetaData meta_tmp, meta_mrg; - if (start == NULL || - PyInt_Check(start) || - PyLong_Check(start) || - PyArray_IsScalar(start, Integer)) { - if (start == NULL) { - start_value = 0; - } - else { - start_value = PyLong_AsLongLong(start); - if (start_value == -1 && PyErr_Occurred()) { - return NULL; - } - } - - meta_value.base = -1; - /* Convert 'stop' to a timedelta */ - if (convert_pyobject_to_timedelta(&meta_value, stop, - &stop_value) < 0) { + /* Get 'start', raising an exception if it is not provided */ + if (start == NULL) { + PyErr_SetString(PyExc_ValueError, + "arange requires both a start and a stop for " + "NumPy datetime64 ranges"); + return NULL; + } + else { + /* Directly copy to meta, no point in gcd for 'start' */ + meta.base = -1; + /* Convert 'start' to a datetime */ + if (convert_pyobject_to_datetime(&meta, start, + &start_value) < 0) { return NULL; } } + + /* Temporary metadata, to merge with 'meta' */ + meta_tmp.base = -1; + /* Convert 'stop' to a datetime */ + if (convert_pyobject_to_datetime(&meta_tmp, stop, + &stop_value) < 0) { + return NULL; + } + + /* Merge the metadata */ + if (compute_datetime_metadata_greatest_common_divisor( + &meta, &meta_tmp, &meta_mrg, 0, 0) < 0) { + return NULL; + } + + /* Cast 'start' and 'stop' to the merged metadata */ + if (cast_datetime_to_datetime(&meta, &meta_mrg, + start_value, &start_value) < 0) { + return NULL; + } + if (cast_datetime_to_datetime(&meta_tmp, &meta_mrg, + stop_value, &stop_value) < 0) { + return NULL; + } + + meta = meta_mrg; + + if (step == NULL) { + step_value = 1; + } else { - meta_tmp1.base = -1; - /* Convert 'start' to a timedelta */ - if (convert_pyobject_to_timedelta(&meta_tmp1, start, - &tmp1) < 0) { + /* Temporary metadata, to merge with 'meta' */ + meta_tmp.base = -1; + /* Convert 'step' to a timedelta */ + if (convert_pyobject_to_timedelta(&meta_tmp, step, + &step_value) < 0) { return NULL; } - if (PyInt_Check(stop) || - PyLong_Check(stop) || - PyArray_IsScalar(stop, Integer)) { - /* Use the metadata and value from 'start' */ - meta_value = meta_tmp1; - start_value = tmp1; - - /* Get the integer value for stop */ - stop_value = PyLong_AsLongLong(stop); - if (stop_value == -1 && PyErr_Occurred()) { - return NULL; - } + /* Merge the metadata */ + if (compute_datetime_metadata_greatest_common_divisor( + &meta, &meta_tmp, &meta_mrg, 0, 1) < 0) { + return NULL; } - /* Otherwise try to interpret 'stop' as a timedelta */ - else { - meta_tmp2.base = -1; - /* Convert to a datetime */ - if (convert_pyobject_to_timedelta(&meta_tmp2, stop, - &tmp2) < 0) { - return NULL; - } - - /* Merge the metadata */ - if (compute_datetime_metadata_greatest_common_divisor( - &meta_tmp1, &meta_tmp2, - &meta_value, 1, 1) < 0) { - return NULL; - } - /* Convert 'start' to the merged metadata */ - if (cast_timedelta_to_timedelta(&meta_tmp1, &meta_value, - tmp1, &start_value) < 0) { - return NULL; - } - - /* Convert 'stop' to the merged metadata */ - if (cast_timedelta_to_timedelta(&meta_tmp2, &meta_value, - tmp2, &stop_value) < 0) { - return NULL; - } + /* Cast 'start', 'stop', and 'step' to the merged metadata */ + if (cast_datetime_to_datetime(&meta, &meta_mrg, + start_value, &start_value) < 0) { + return NULL; + } + if (cast_datetime_to_datetime(&meta, &meta_mrg, + stop_value, &stop_value) < 0) { + return NULL; } + if (cast_timedelta_to_timedelta(&meta_tmp, &meta_mrg, + step_value, &step_value) < 0) { + return NULL; + } + + meta = meta_mrg; } } } From 53ab0c1fd5cc4a4b49e5a6df3450be15289eb6a3 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Thu, 9 Jun 2011 11:39:13 -0500 Subject: [PATCH 09/34] ENH: datetime-arange: The arange function largely works now --- numpy/core/src/multiarray/_datetime.h | 2 +- numpy/core/src/multiarray/ctors.c | 2 +- numpy/core/src/multiarray/datetime.c | 198 ++++++++++++++++++-------- numpy/core/tests/test_datetime.py | 14 ++ 4 files changed, 158 insertions(+), 58 deletions(-) diff --git a/numpy/core/src/multiarray/_datetime.h b/numpy/core/src/multiarray/_datetime.h index 34b3b5eba75f..c2645b34d922 100644 --- a/numpy/core/src/multiarray/_datetime.h +++ b/numpy/core/src/multiarray/_datetime.h @@ -376,7 +376,7 @@ is_any_numpy_datetime_or_timedelta(PyObject *obj); /* * Implements a datetime-specific arange */ -NPY_NO_EXPORT PyObject * +NPY_NO_EXPORT PyArrayObject * datetime_arange(PyObject *start, PyObject *stop, PyObject *step, PyArray_Descr *dtype); diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index b400f1feccab..35ed706c4b4c 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -3085,7 +3085,7 @@ PyArray_ArangeObj(PyObject *start, PyObject *stop, PyObject *step, PyArray_Descr (dtype == NULL && (is_any_numpy_datetime_or_timedelta(start) || is_any_numpy_datetime_or_timedelta(stop) || is_any_numpy_datetime_or_timedelta(step)))) { - return datetime_arange(start, stop, step, dtype); + return (PyObject *)datetime_arange(start, stop, step, dtype); } if (!dtype) { diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 3a208bb3b850..6c41fe708476 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -1178,30 +1178,70 @@ parse_datetime_metadata_from_metastr(char *metastr, Py_ssize_t len, return -1; } -NPY_NO_EXPORT PyObject * -parse_datetime_metacobj_from_metastr(char *metastr, Py_ssize_t len) +/* + * Creates a datetime or timedelta dtype using the provided metadata. + */ +NPY_NO_EXPORT PyArray_Descr * +create_datetime_dtype(int type_num, PyArray_DatetimeMetaData *meta) { + PyArray_Descr *dtype = NULL; PyArray_DatetimeMetaData *dt_data; + PyObject *metacobj = NULL; + + /* Create a default datetime or timedelta */ + if (type_num == NPY_DATETIME || type_num == NPY_TIMEDELTA) { + dtype = PyArray_DescrNewFromType(type_num); + } + else { + PyErr_SetString(PyExc_RuntimeError, + "Asked to create a datetime type with a non-datetime " + "type number"); + return NULL; + } + + if (dtype == NULL) { + return NULL; + } + + /* + * Remove any reference to old metadata dictionary + * And create a new one for this new dtype + */ + Py_XDECREF(dtype->metadata); + dtype->metadata = PyDict_New(); + if (dtype->metadata == NULL) { + Py_DECREF(dtype); + return NULL; + } + /* Create a metadata capsule to copy the provided metadata */ dt_data = PyArray_malloc(sizeof(PyArray_DatetimeMetaData)); if (dt_data == NULL) { - return PyErr_NoMemory(); + Py_DECREF(dtype); + PyErr_NoMemory(); + return NULL; } - /* If there's no metastr, use the default */ - if (len == 0) { - dt_data->base = NPY_DATETIME_DEFAULTUNIT; - dt_data->num = 1; - dt_data->events = 1; + /* Copy the metadata */ + *dt_data = *meta; + + /* Allocate a capsule for it (this claims ownership of dt_data) */ + metacobj = NpyCapsule_FromVoidPtr((void *)dt_data, simple_capsule_dtor); + if (metacobj == NULL) { + Py_DECREF(dtype); + return NULL; } - else { - if (parse_datetime_metadata_from_metastr(metastr, len, dt_data) < 0) { - PyArray_free(dt_data); - return NULL; - } + + /* Set the metadata object in the dictionary. */ + if (PyDict_SetItemString(dtype->metadata, NPY_METADATA_DTSTR, + metacobj) < 0) { + Py_DECREF(dtype); + Py_DECREF(metacobj); + return NULL; } + Py_DECREF(metacobj); - return NpyCapsule_FromVoidPtr((void *)dt_data, simple_capsule_dtor); + return dtype; } /* @@ -1211,11 +1251,10 @@ parse_datetime_metacobj_from_metastr(char *metastr, Py_ssize_t len) NPY_NO_EXPORT PyArray_Descr * parse_dtype_from_datetime_typestr(char *typestr, Py_ssize_t len) { - PyArray_Descr *dtype = NULL; + PyArray_DatetimeMetaData meta; char *metastr = NULL; int is_timedelta = 0; Py_ssize_t metalen = 0; - PyObject *metacobj = NULL; if (len < 2) { PyErr_Format(PyExc_TypeError, @@ -1255,45 +1294,13 @@ parse_dtype_from_datetime_typestr(char *typestr, Py_ssize_t len) return NULL; } - /* Create a default datetime or timedelta */ - if (is_timedelta) { - dtype = PyArray_DescrNewFromType(NPY_TIMEDELTA); - } - else { - dtype = PyArray_DescrNewFromType(NPY_DATETIME); - } - if (dtype == NULL) { + /* Parse the metadata string into a metadata struct */ + if (parse_datetime_metadata_from_metastr(metastr, metalen, &meta) < 0) { return NULL; } - /* - * Remove any reference to old metadata dictionary - * And create a new one for this new dtype - */ - Py_XDECREF(dtype->metadata); - dtype->metadata = PyDict_New(); - if (dtype->metadata == NULL) { - Py_DECREF(dtype); - return NULL; - } - - /* Parse the metadata string into a metadata capsule */ - metacobj = parse_datetime_metacobj_from_metastr(metastr, metalen); - if (metacobj == NULL) { - Py_DECREF(dtype); - return NULL; - } - - /* Set the metadata object in the dictionary. */ - if (PyDict_SetItemString(dtype->metadata, NPY_METADATA_DTSTR, - metacobj) < 0) { - Py_DECREF(dtype); - Py_DECREF(metacobj); - return NULL; - } - Py_DECREF(metacobj); - - return dtype; + return create_datetime_dtype(is_timedelta ? NPY_TIMEDELTA : NPY_DATETIME, + &meta); } static NPY_DATETIMEUNIT _multiples_table[16][4] = { @@ -4445,7 +4452,7 @@ is_any_numpy_datetime_or_timedelta(PyObject *obj) is_any_numpy_timedelta(obj)); } -NPY_NO_EXPORT PyObject * +NPY_NO_EXPORT PyArrayObject * datetime_arange(PyObject *start, PyObject *stop, PyObject *step, PyArray_Descr *dtype) { @@ -4457,6 +4464,10 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, */ npy_int64 start_value = 0, stop_value = 0, step_value; + npy_intp i, length; + PyArrayObject *ret; + npy_int64 *ret_data; + /* * First normalize the input parameters so there is no Py_None, * and start is moved to stop if stop is unspecified. @@ -4562,6 +4573,17 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, } } } + + /* Get the 'step' value */ + if (step == NULL) { + step_value = 1; + } + else { + if (convert_pyobject_to_timedelta(&meta, step, + &step_value) < 0) { + return NULL; + } + } } /* if dtype is NULL */ else { @@ -4735,7 +4757,71 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, } } } - - PyErr_SetString(PyExc_RuntimeError, "datetime_arange is incomplete"); - return NULL; + + /* Now start, stop, and step have their values and matching metadata */ + if (start_value == NPY_DATETIME_NAT || + stop_value == NPY_DATETIME_NAT || + step_value == NPY_DATETIME_NAT) { + PyErr_SetString(PyExc_ValueError, + "arange: cannot use NaT (not-a-time) datetime values"); + return NULL; + } + + /* Calculate the array length */ + if (step_value > 0) { + if (stop_value > start_value) { + length = (stop_value - start_value) / step_value; + } + else { + length = 0; + } + } + else if (step_value < 0) { + if (stop_value < start_value) { + length = (start_value - stop_value) / (-step_value); + } + else { + length = 0; + } + } + else { + PyErr_SetString(PyExc_ValueError, + "arange: step may not be zero"); + return NULL; + } + + /* Create the dtype of the result */ + if (dtype != NULL) { + Py_INCREF(dtype); + } + else { + dtype = create_datetime_dtype( + is_timedelta ? NPY_TIMEDELTA : NPY_DATETIME, + &meta); + if (dtype == NULL) { + return NULL; + } + } + + /* Create the result array */ + ret = (PyArrayObject *)PyArray_NewFromDescr( + &PyArray_Type, dtype, 1, &length, NULL, + NULL, 0, NULL); + if (ret == NULL) { + return NULL; + } + + if (length > 0) { + /* Extract the data pointer */ + ret_data = (npy_int64 *)PyArray_DATA(ret); + + /* Create the timedeltas or datetimes */ + for (i = 0; i < length; ++i) { + *ret_data = start_value; + start_value += step_value; + ret_data++; + } + } + + return ret; } diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index 2d13d399e579..9470c16d7d80 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -1100,6 +1100,20 @@ def test_datetime_as_string(self): assert_equal(np.datetime_as_string(a, local=True, tzoffset=-5*60), '2010-03-15T01:30-0500') + def test_datetime_arange(self): + # Default mode + a = np.arange('2010-01-05', '2010-01-10', dtype='M8[D]') + assert_equal(a.dtype, np.dtype('M8[D]')) + assert_equal(a, + np.array(['2010-01-05', '2010-01-06', '2010-01-07', + '2010-01-08', '2010-01-09'], dtype='M8[D]')) + + a = np.arange('1950-02-10', '1950-02-06', -1, dtype='M8[D]') + assert_equal(a.dtype, np.dtype('M8[D]')) + assert_equal(a, + np.array(['1950-02-10', '1950-02-09', '1950-02-08', + '1950-02-07'], dtype='M8[D]')) + class TestDateTimeData(TestCase): def test_basic(self): From 98b4c3812a822669790ad2107ce72de30d4c92a4 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Thu, 9 Jun 2011 12:02:04 -0500 Subject: [PATCH 10/34] ENH: datetime-arange: Detect the unit when a dtype with generic units is given --- numpy/core/src/multiarray/datetime.c | 49 +++++++++++++++++++--------- numpy/core/tests/test_datetime.py | 8 ++++- 2 files changed, 40 insertions(+), 17 deletions(-) diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 6c41fe708476..606d69c50383 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -4457,7 +4457,7 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, PyArray_Descr *dtype) { PyArray_DatetimeMetaData meta; - int is_timedelta = 0; + int type_num = 0; /* * Both datetime and timedelta are stored as int64, so they can * share value variables. @@ -4496,17 +4496,14 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, return NULL; } - /* - * Now figure out whether we're doing a datetime or a timedelta - * arange, and get the metadata for start and stop. + /* Check if the units of the given dtype are generic, in which + * case we use the code path that detects the units */ if (dtype != NULL) { PyArray_DatetimeMetaData *meta_tmp; - if (dtype->type_num == NPY_TIMEDELTA) { - is_timedelta = 1; - } - else if (dtype->type_num != NPY_DATETIME) { + type_num = dtype->type_num; + if (type_num != NPY_DATETIME && type_num != NPY_TIMEDELTA) { PyErr_SetString(PyExc_ValueError, "datetime_arange was given a non-datetime dtype"); return NULL; @@ -4517,9 +4514,33 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, return NULL; } - meta = *meta_tmp; + /* + * If the dtype specified is in generic units, detect the + * units from the input parameters. + */ + if (meta_tmp->base == NPY_FR_GENERIC) { + dtype = NULL; + } + /* Otherwise use the provided metadata */ + else { + meta = *meta_tmp; + } + } + else { + if (is_any_numpy_datetime(start) || is_any_numpy_datetime(stop)) { + type_num = NPY_DATETIME; + } + else { + type_num = NPY_TIMEDELTA; + } + } - if (is_timedelta) { + /* + * Now figure out whether we're doing a datetime or a timedelta + * arange, and get the metadata for start and stop. + */ + if (dtype != NULL) { + if (type_num == NPY_TIMEDELTA) { /* Get 'start', defaulting to 0 if not provided */ if (start == NULL) { start_value = 0; @@ -4594,11 +4615,9 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, meta.base = NPY_FR_GENERIC; /* Timedelta arange */ - if (!is_any_numpy_datetime(start) && !is_any_numpy_datetime(stop)) { + if (type_num == NPY_TIMEDELTA) { PyArray_DatetimeMetaData meta_tmp, meta_mrg; - is_timedelta = 1; - /* Convert start to a timedelta */ if (start == NULL) { start_value = 0; @@ -4795,9 +4814,7 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, Py_INCREF(dtype); } else { - dtype = create_datetime_dtype( - is_timedelta ? NPY_TIMEDELTA : NPY_DATETIME, - &meta); + dtype = create_datetime_dtype(type_num, &meta); if (dtype == NULL) { return NULL; } diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index 9470c16d7d80..a8f1be8fc533 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -1101,7 +1101,7 @@ def test_datetime_as_string(self): '2010-03-15T01:30-0500') def test_datetime_arange(self): - # Default mode + # With two datetimes provided as strings a = np.arange('2010-01-05', '2010-01-10', dtype='M8[D]') assert_equal(a.dtype, np.dtype('M8[D]')) assert_equal(a, @@ -1114,6 +1114,12 @@ def test_datetime_arange(self): np.array(['1950-02-10', '1950-02-09', '1950-02-08', '1950-02-07'], dtype='M8[D]')) + # Unit should be detected as months here + a = np.arange('1969-05', '1970-05', 2, dtype='M8') + assert_equal(a.dtype, np.dtype('M8[M]')) + assert_equal(a, + np.datetime64('1969-05') + np.arange(12, step=2)) + class TestDateTimeData(TestCase): def test_basic(self): From dadf6c21c9880244188a1ebe5ebf55c5bde920f8 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Thu, 9 Jun 2011 14:36:32 -0500 Subject: [PATCH 11/34] ENH: datetime-arange: Move the unit metadata promotion to a separate function This cleans up the implementation of arange a lot, and makes the promotion rules behave consistently. --- numpy/core/src/multiarray/datetime.c | 454 ++++++++++++--------------- numpy/core/tests/test_datetime.py | 39 +++ 2 files changed, 235 insertions(+), 258 deletions(-) diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 606d69c50383..04f68077ca31 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -4452,17 +4452,154 @@ is_any_numpy_datetime_or_timedelta(PyObject *obj) is_any_numpy_timedelta(obj)); } +/* + * Converts an array of PyObject * into datetimes and/or timedeltas, + * based on the values in type_nums. + * + * If inout_meta->base is -1, uses GCDs to calculate the metadata, filling + * in 'inout_meta' with the resulting metadata. Otherwise uses the provided + * 'inout_meta' for all the conversions. + * + * When obj[i] is NULL, out_value[i] will be set to NPY_DATETIME_NAT. + * + * Returns 0 on success, -1 on failure. + */ +NPY_NO_EXPORT int +convert_pyobjects_to_datetimes(int count, + PyObject **objs, int *type_nums, + npy_int64 *out_values, + PyArray_DatetimeMetaData *inout_meta) +{ + int i, is_out_strict; + PyArray_DatetimeMetaData *meta; + + /* No values trivially succeeds */ + if (count == 0) { + return 0; + } + + /* Use the inputs to resolve the unit metadata if requested */ + if (inout_meta->base == -1) { + /* Allocate an array of metadata corresponding to the objects */ + meta = PyArray_malloc(count * sizeof(PyArray_DatetimeMetaData)); + if (meta == NULL) { + PyErr_NoMemory(); + return -1; + } + + /* Convert all the objects into timedeltas or datetimes */ + for (i = 0; i < count; ++i) { + meta[i].base = -1; + meta[i].num = 1; + meta[i].events = 1; + + /* NULL -> NaT */ + if (objs[i] == NULL) { + out_values[i] = NPY_DATETIME_NAT; + meta[i].base = NPY_FR_GENERIC; + } + else if (type_nums[i] == NPY_DATETIME) { + if (convert_pyobject_to_datetime(&meta[i], objs[i], + &out_values[i]) < 0) { + PyArray_free(meta); + return -1; + } + } + else if (type_nums[i] == NPY_TIMEDELTA) { + if (convert_pyobject_to_timedelta(&meta[i], objs[i], + &out_values[i]) < 0) { + PyArray_free(meta); + return -1; + } + } + else { + PyErr_SetString(PyExc_ValueError, + "convert_pyobjects_to_datetimes requires that " + "all the type_nums provided be datetime or timedelta"); + PyArray_free(meta); + return -1; + } + } + + /* Merge all the metadatas, starting with the first one */ + *inout_meta = meta[0]; + is_out_strict = (type_nums[0] == NPY_TIMEDELTA); + + for (i = 1; i < count; ++i) { + if (compute_datetime_metadata_greatest_common_divisor( + &meta[i], inout_meta, inout_meta, + type_nums[i] == NPY_TIMEDELTA, + is_out_strict) < 0) { + PyArray_free(meta); + return -1; + } + is_out_strict = is_out_strict || (type_nums[i] == NPY_TIMEDELTA); + } + + /* Convert all the values into the resolved unit metadata */ + for (i = 0; i < count; ++i) { + if (type_nums[i] == NPY_DATETIME) { + if (cast_datetime_to_datetime(&meta[i], inout_meta, + out_values[i], &out_values[i]) < 0) { + PyArray_free(meta); + return -1; + } + } + else if (type_nums[i] == NPY_TIMEDELTA) { + if (cast_timedelta_to_timedelta(&meta[i], inout_meta, + out_values[i], &out_values[i]) < 0) { + PyArray_free(meta); + return -1; + } + } + } + + PyArray_free(meta); + } + /* Otherwise convert to the provided unit metadata */ + else { + /* Convert all the objects into timedeltas or datetimes */ + for (i = 0; i < count; ++i) { + /* NULL -> NaT */ + if (objs[i] == NULL) { + out_values[i] = NPY_DATETIME_NAT; + } + else if (type_nums[i] == NPY_DATETIME) { + if (convert_pyobject_to_datetime(inout_meta, objs[i], + &out_values[i]) < 0) { + return -1; + } + } + else if (type_nums[i] == NPY_TIMEDELTA) { + if (convert_pyobject_to_timedelta(inout_meta, objs[i], + &out_values[i]) < 0) { + return -1; + } + } + else { + PyErr_SetString(PyExc_ValueError, + "convert_pyobjects_to_datetimes requires that " + "all the type_nums provided be datetime or timedelta"); + return -1; + } + } + } + + return 0; +} + NPY_NO_EXPORT PyArrayObject * datetime_arange(PyObject *start, PyObject *stop, PyObject *step, PyArray_Descr *dtype) { PyArray_DatetimeMetaData meta; - int type_num = 0; /* * Both datetime and timedelta are stored as int64, so they can * share value variables. */ - npy_int64 start_value = 0, stop_value = 0, step_value; + npy_int64 values[3]; + PyObject *objs[3]; + int type_nums[3]; npy_intp i, length; PyArrayObject *ret; @@ -4502,8 +4639,8 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, if (dtype != NULL) { PyArray_DatetimeMetaData *meta_tmp; - type_num = dtype->type_num; - if (type_num != NPY_DATETIME && type_num != NPY_TIMEDELTA) { + type_nums[0] = dtype->type_num; + if (type_nums[0] != NPY_DATETIME && type_nums[0] != NPY_TIMEDELTA) { PyErr_SetString(PyExc_ValueError, "datetime_arange was given a non-datetime dtype"); return NULL; @@ -4520,6 +4657,7 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, */ if (meta_tmp->base == NPY_FR_GENERIC) { dtype = NULL; + meta.base = -1; } /* Otherwise use the provided metadata */ else { @@ -4528,280 +4666,80 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, } else { if (is_any_numpy_datetime(start) || is_any_numpy_datetime(stop)) { - type_num = NPY_DATETIME; + type_nums[0] = NPY_DATETIME; } else { - type_num = NPY_TIMEDELTA; + type_nums[0] = NPY_TIMEDELTA; } - } - /* - * Now figure out whether we're doing a datetime or a timedelta - * arange, and get the metadata for start and stop. - */ - if (dtype != NULL) { - if (type_num == NPY_TIMEDELTA) { - /* Get 'start', defaulting to 0 if not provided */ - if (start == NULL) { - start_value = 0; - } - else { - if (convert_pyobject_to_timedelta(&meta, start, - &start_value) < 0) { - return NULL; - } - } - /* Get 'stop' */ - if (convert_pyobject_to_timedelta(&meta, stop, - &stop_value) < 0) { - return NULL; - } - } - else { - /* Get 'start', raising an exception if it is not provided */ - if (start == NULL) { - PyErr_SetString(PyExc_ValueError, - "arange requires both a start and a stop for " - "NumPy datetime64 ranges"); - return NULL; - } - else { - if (convert_pyobject_to_datetime(&meta, start, - &start_value) < 0) { - return NULL; - } - } - /* - * Get 'stop', treating it specially if it's an integer - * or a timedelta. - */ - if (PyInt_Check(stop) || - PyLong_Check(stop) || - PyArray_IsScalar(stop, Integer) || - is_any_numpy_timedelta(stop)) { - if (convert_pyobject_to_timedelta(&meta, stop, - &stop_value) < 0) { - return NULL; - } + meta.base = -1; + } - /* Add the start value to make it a datetime */ - stop_value += start_value; - } - else { - if (convert_pyobject_to_datetime(&meta, stop, - &stop_value) < 0) { - return NULL; - } - } - } + if (type_nums[0] == NPY_DATETIME && start == NULL) { + PyErr_SetString(PyExc_ValueError, + "arange requires both a start and a stop for " + "NumPy datetime64 ranges"); + return NULL; + } - /* Get the 'step' value */ - if (step == NULL) { - step_value = 1; - } - else { - if (convert_pyobject_to_timedelta(&meta, step, - &step_value) < 0) { - return NULL; - } - } + /* Set up to convert the objects to a common datetime unit metadata */ + objs[0] = start; + objs[1] = stop; + objs[2] = step; + if (type_nums[0] == NPY_TIMEDELTA) { + type_nums[1] = NPY_TIMEDELTA; + type_nums[2] = NPY_TIMEDELTA; } - /* if dtype is NULL */ else { - /* - * Start meta as generic units, so taking the gcd - * does the right thing - */ - meta.base = NPY_FR_GENERIC; - - /* Timedelta arange */ - if (type_num == NPY_TIMEDELTA) { - PyArray_DatetimeMetaData meta_tmp, meta_mrg; - - /* Convert start to a timedelta */ - if (start == NULL) { - start_value = 0; - } - else { - /* Directly copy to meta, no point in gcd for 'start' */ - meta.base = -1; - /* Convert 'start' to a timedelta */ - if (convert_pyobject_to_timedelta(&meta, start, - &start_value) < 0) { - return NULL; - } - } - - /* Temporary metadata, to merge with 'meta' */ - meta_tmp.base = -1; - /* Convert 'stop' to a timedelta */ - if (convert_pyobject_to_timedelta(&meta_tmp, stop, - &stop_value) < 0) { - return NULL; - } - - /* Merge the metadata */ - if (compute_datetime_metadata_greatest_common_divisor( - &meta, &meta_tmp, &meta_mrg, 1, 1) < 0) { - return NULL; - } - - /* Cast 'start' and 'stop' to the merged metadata */ - if (cast_timedelta_to_timedelta(&meta, &meta_mrg, - start_value, &start_value) < 0) { - return NULL; - } - if (cast_timedelta_to_timedelta(&meta_tmp, &meta_mrg, - stop_value, &stop_value) < 0) { - return NULL; - } - - meta = meta_mrg; - - if (step == NULL) { - step_value = 1; - } - else { - /* Temporary metadata, to merge with 'meta' */ - meta_tmp.base = -1; - /* Convert 'step' to a timedelta */ - if (convert_pyobject_to_timedelta(&meta_tmp, step, - &step_value) < 0) { - return NULL; - } - - /* Merge the metadata */ - if (compute_datetime_metadata_greatest_common_divisor( - &meta, &meta_tmp, &meta_mrg, 1, 1) < 0) { - return NULL; - } - - /* Cast 'start', 'stop', and 'step' to the merged metadata */ - if (cast_timedelta_to_timedelta(&meta, &meta_mrg, - start_value, &start_value) < 0) { - return NULL; - } - if (cast_timedelta_to_timedelta(&meta, &meta_mrg, - stop_value, &stop_value) < 0) { - return NULL; - } - if (cast_timedelta_to_timedelta(&meta_tmp, &meta_mrg, - step_value, &step_value) < 0) { - return NULL; - } - - meta = meta_mrg; - } + if (PyInt_Check(objs[1]) || + PyLong_Check(objs[1]) || + PyArray_IsScalar(objs[1], Integer) || + is_any_numpy_timedelta(objs[1])) { + type_nums[1] = NPY_TIMEDELTA; } - /* Datetime arange */ else { - PyArray_DatetimeMetaData meta_tmp, meta_mrg; - - /* Get 'start', raising an exception if it is not provided */ - if (start == NULL) { - PyErr_SetString(PyExc_ValueError, - "arange requires both a start and a stop for " - "NumPy datetime64 ranges"); - return NULL; - } - else { - /* Directly copy to meta, no point in gcd for 'start' */ - meta.base = -1; - /* Convert 'start' to a datetime */ - if (convert_pyobject_to_datetime(&meta, start, - &start_value) < 0) { - return NULL; - } - } - - /* Temporary metadata, to merge with 'meta' */ - meta_tmp.base = -1; - /* Convert 'stop' to a datetime */ - if (convert_pyobject_to_datetime(&meta_tmp, stop, - &stop_value) < 0) { - return NULL; - } - - /* Merge the metadata */ - if (compute_datetime_metadata_greatest_common_divisor( - &meta, &meta_tmp, &meta_mrg, 0, 0) < 0) { - return NULL; - } - - /* Cast 'start' and 'stop' to the merged metadata */ - if (cast_datetime_to_datetime(&meta, &meta_mrg, - start_value, &start_value) < 0) { - return NULL; - } - if (cast_datetime_to_datetime(&meta_tmp, &meta_mrg, - stop_value, &stop_value) < 0) { - return NULL; - } - - meta = meta_mrg; - - if (step == NULL) { - step_value = 1; - } - else { - /* Temporary metadata, to merge with 'meta' */ - meta_tmp.base = -1; - /* Convert 'step' to a timedelta */ - if (convert_pyobject_to_timedelta(&meta_tmp, step, - &step_value) < 0) { - return NULL; - } + type_nums[1] = NPY_DATETIME; + } + type_nums[2] = NPY_TIMEDELTA; + } - /* Merge the metadata */ - if (compute_datetime_metadata_greatest_common_divisor( - &meta, &meta_tmp, &meta_mrg, 0, 1) < 0) { - return NULL; - } + /* Convert all the arguments */ + if (convert_pyobjects_to_datetimes(3, objs, type_nums, + values, &meta) < 0) { + return NULL; + } - /* Cast 'start', 'stop', and 'step' to the merged metadata */ - if (cast_datetime_to_datetime(&meta, &meta_mrg, - start_value, &start_value) < 0) { - return NULL; - } - if (cast_datetime_to_datetime(&meta, &meta_mrg, - stop_value, &stop_value) < 0) { - return NULL; - } - if (cast_timedelta_to_timedelta(&meta_tmp, &meta_mrg, - step_value, &step_value) < 0) { - return NULL; - } + /* If no step was provided, default to 1 */ + if (step == NULL) { + values[2] = 1; + } - meta = meta_mrg; - } - } + /* + * In the case of arange(datetime, timedelta), convert + * the timedelta into a datetime by adding the start datetime. + */ + if (type_nums[0] == NPY_DATETIME && type_nums[1] == NPY_TIMEDELTA) { + values[1] += values[0]; } /* Now start, stop, and step have their values and matching metadata */ - if (start_value == NPY_DATETIME_NAT || - stop_value == NPY_DATETIME_NAT || - step_value == NPY_DATETIME_NAT) { + if (values[0] == NPY_DATETIME_NAT || + values[1] == NPY_DATETIME_NAT || + values[2] == NPY_DATETIME_NAT) { PyErr_SetString(PyExc_ValueError, "arange: cannot use NaT (not-a-time) datetime values"); return NULL; } /* Calculate the array length */ - if (step_value > 0) { - if (stop_value > start_value) { - length = (stop_value - start_value) / step_value; - } - else { - length = 0; - } + if (values[2] > 0 && values[1] > values[0]) { + length = (values[1] - values[0] + (values[2] - 1)) / values[2]; } - else if (step_value < 0) { - if (stop_value < start_value) { - length = (start_value - stop_value) / (-step_value); - } - else { - length = 0; - } + else if (values[2] < 0 && values[1] < values[0]) { + length = (values[1] - values[0] + (values[2] + 1)) / values[2]; + } + else if (values[2] != 0) { + length = 0; } else { PyErr_SetString(PyExc_ValueError, @@ -4814,7 +4752,7 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, Py_INCREF(dtype); } else { - dtype = create_datetime_dtype(type_num, &meta); + dtype = create_datetime_dtype(type_nums[0], &meta); if (dtype == NULL) { return NULL; } @@ -4834,8 +4772,8 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, /* Create the timedeltas or datetimes */ for (i = 0; i < length; ++i) { - *ret_data = start_value; - start_value += step_value; + *ret_data = values[0]; + values[0] += values[2]; ret_data++; } } diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index a8f1be8fc533..f75b5c1a93dc 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -1120,6 +1120,45 @@ def test_datetime_arange(self): assert_equal(a, np.datetime64('1969-05') + np.arange(12, step=2)) + # datetime, integer|timedelta works as well + # produces arange (start, start + stop) in this case + a = np.arange('1969', 18, 3, dtype='M8') + assert_equal(a.dtype, np.dtype('M8[Y]')) + assert_equal(a, + np.datetime64('1969') + np.arange(18, step=3)) + a = np.arange('1969-12-19', 22, np.timedelta64(2), dtype='M8') + assert_equal(a.dtype, np.dtype('M8[D]')) + assert_equal(a, + np.datetime64('1969-12-19') + np.arange(22, step=2)) + + # Step of 0 is disallowed + assert_raises(ValueError, np.arange, np.datetime64('today'), + np.datetime64('today') + 3, 0) + # Promotion across nonlinear unit boundaries is disallowed + assert_raises(TypeError, np.arange, np.datetime64('2011-03-01','D'), + np.timedelta64(5,'M')) + assert_raises(TypeError, np.arange, + np.datetime64('2012-02-03T14Z','s'), + np.timedelta64(5,'Y')) + + def test_timedelta_arange(self): + a = np.arange(3, 10, dtype='m8') + assert_equal(a.dtype, np.dtype('m8')) + assert_equal(a, np.timedelta64(0) + np.arange(3, 10)) + + a = np.arange(np.timedelta64(3,'s'), 10, 2, dtype='m8') + assert_equal(a.dtype, np.dtype('m8[s]')) + assert_equal(a, np.timedelta64(0, 's') + np.arange(3, 10, 2)) + + # Step of 0 is disallowed + assert_raises(ValueError, np.arange, np.timedelta64(0), + np.timedelta64(5), 0) + # Promotion across nonlinear unit boundaries is disallowed + assert_raises(TypeError, np.arange, np.timedelta64(0,'D'), + np.timedelta64(5,'M')) + assert_raises(TypeError, np.arange, np.timedelta64(0,'Y'), + np.timedelta64(5,'D')) + class TestDateTimeData(TestCase): def test_basic(self): From c3f963e6d48c9fe06a4face4518dba89e666243d Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Thu, 9 Jun 2011 17:57:15 -0500 Subject: [PATCH 12/34] ENH: datetime-bday: Remove business days as a datetime metadata unit The complexity of the operations desired for business days is such that expressing it as a unit in the datetime doesn't fit naturally. Instead, an API operating on day-based datetimes appears to be a superior approach. --- numpy/core/include/numpy/ndarraytypes.h | 2 - numpy/core/src/multiarray/datetime.c | 147 ++------------------ numpy/core/src/multiarray/dtype_transfer.c | 11 +- numpy/core/src/multiarray/scalartypes.c.src | 1 - numpy/core/tests/test_datetime.py | 15 +- 5 files changed, 17 insertions(+), 159 deletions(-) diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index 7d04c0106ba9..de6385ded502 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -233,7 +233,6 @@ typedef enum { NPY_FR_Y, /* Years */ NPY_FR_M, /* Months */ NPY_FR_W, /* Weeks */ - NPY_FR_B, /* Business days (weekdays, doesn't account for holidays) */ NPY_FR_D, /* Days */ NPY_FR_h, /* hours */ NPY_FR_m, /* minutes */ @@ -253,7 +252,6 @@ typedef enum { #define NPY_STR_Y "Y" #define NPY_STR_M "M" #define NPY_STR_W "W" -#define NPY_STR_B "B" #define NPY_STR_D "D" #define NPY_STR_h "h" #define NPY_STR_m "m" diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 04f68077ca31..881c0f305f98 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -32,7 +32,6 @@ NPY_NO_EXPORT char *_datetime_strings[] = { NPY_STR_Y, NPY_STR_M, NPY_STR_W, - NPY_STR_B, NPY_STR_D, NPY_STR_h, NPY_STR_m, @@ -46,50 +45,12 @@ NPY_NO_EXPORT char *_datetime_strings[] = { "generic" }; -/* - ==================================================== - } - == Beginning of section borrowed from mx.DateTime == - ==================================================== -*/ - -/* - * Functions in the following section are borrowed from mx.DateTime version - * 2.0.6, and hence this code is subject to the terms of the egenix public - * license version 1.0.0 - */ - -/* Table of number of days in a month (0-based, without and with leap) */ +/* Days per month, regular year and leap year */ static int days_in_month[2][12] = { { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }, { 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 } }; -/* - * Return the day of the week for the given absolute date. - * Monday is 0 and Sunday is 6 - */ -static int -day_of_week(npy_longlong absdate) -{ - /* Add in four for the Thursday on Jan 1, 1970 (epoch offset)*/ - absdate += 4; - - if (absdate >= 0) { - return absdate % 7; - } - else { - return 6 + (absdate + 1) % 7; - } -} - -/* - ==================================================== - == End of section adapted from mx.DateTime == - ==================================================== -*/ - - static int is_leapyear(npy_int64 year) { @@ -274,7 +235,6 @@ convert_datetimestruct_to_datetime(PyArray_DatetimeMetaData *meta, else { /* Otherwise calculate the number of days to start */ npy_int64 days = get_datetimestruct_days(dts); - int dotw; switch (base) { case NPY_FR_W: @@ -286,26 +246,6 @@ convert_datetimestruct_to_datetime(PyArray_DatetimeMetaData *meta, ret = (days - 6) / 7; } break; - case NPY_FR_B: - /* TODO: this needs work... */ - dotw = day_of_week(days); - - if (dotw > 4) { - /* Invalid business day */ - ret = 0; - } - else { - npy_int64 x; - if (days >= 0) { - /* offset to adjust first week */ - x = days - 4; - } - else { - x = days - 2; - } - ret = 2 + (x / 7) * 5 + x % 7; - } - break; case NPY_FR_D: ret = days; break; @@ -457,15 +397,6 @@ PyArray_TimedeltaStructToTimedelta(NPY_DATETIMEUNIT fr, npy_timedeltastruct *d) ret = (d->day - 6) / 7; } } - else if (fr == NPY_FR_B) { - /* - * What is the meaning of a relative Business day? - * - * This assumes you want to take the day difference and - * convert it to business-day difference (removing 2 every 7). - */ - ret = (d->day / 7) * 5 + d->day % 7; - } else if (fr == NPY_FR_D) { ret = d->day; } @@ -536,7 +467,6 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, npy_datetime dt, npy_datetimestruct *out) { - npy_int64 absdays; npy_int64 perday; /* Initialize the output to all zeros */ @@ -597,28 +527,6 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, set_datetimestruct_days(dt * 7, out); break; - case NPY_FR_B: - /* TODO: fix up business days */ - /* Number of business days since Thursday, 1-1-70 */ - /* - * A business day is M T W Th F (i.e. all but Sat and Sun.) - * Convert the business day to the number of actual days. - * - * Must convert [0,1,2,3,4,5,6,7,...] to - * [0,1,4,5,6,7,8,11,...] - * and [...,-9,-8,-7,-6,-5,-4,-3,-2,-1,0] to - * [...,-13,-10,-9,-8,-7,-6,-3,-2,-1,0] - */ - if (dt >= 0) { - absdays = 7 * ((dt + 3) / 5) + ((dt + 3) % 5) - 3; - } - else { - /* Recall how C computes / and % with negative numbers */ - absdays = 7 * ((dt - 1) / 5) + ((dt - 1) % 5) + 1; - } - set_datetimestruct_days(absdays, out); - break; - case NPY_FR_D: set_datetimestruct_days(dt, out); break; @@ -828,7 +736,7 @@ PyArray_DatetimeToDatetimeStruct(npy_datetime val, NPY_DATETIMEUNIT fr, /* * FIXME: Overflow is not handled at all - * To convert from Years, Months, and Business Days, + * To convert from Years or Months, * multiplication by the average is done */ @@ -865,10 +773,6 @@ PyArray_TimedeltaToTimedeltaStruct(npy_timedelta val, NPY_DATETIMEUNIT fr, else if (fr == NPY_FR_W) { day = val * 7; } - else if (fr == NPY_FR_B) { - /* Number of business days since Thursday, 1-1-70 */ - day = (val * 7) / 5; - } else if (fr == NPY_FR_D) { day = val; } @@ -1308,10 +1212,8 @@ static NPY_DATETIMEUNIT _multiples_table[16][4] = { {NPY_FR_M, NPY_FR_W, NPY_FR_D}, {4, 30, 720}, /* NPY_FR_M */ {NPY_FR_W, NPY_FR_D, NPY_FR_h}, - {5, 7, 168, 10080}, /* NPY_FR_W */ - {NPY_FR_B, NPY_FR_D, NPY_FR_h, NPY_FR_m}, - {24, 1440, 86400}, /* NPY_FR_B */ - {NPY_FR_h, NPY_FR_m, NPY_FR_s}, + {7, 168, 10080}, /* NPY_FR_W */ + {NPY_FR_D, NPY_FR_h, NPY_FR_m}, {24, 1440, 86400}, /* NPY_FR_D */ {NPY_FR_h, NPY_FR_m, NPY_FR_s}, {60, 3600}, /* NPY_FR_h */ @@ -1401,14 +1303,13 @@ convert_datetime_divisor_to_multiple(PyArray_DatetimeMetaData *meta, /* * Lookup table for factors between datetime units, except - * for years, months, and business days. + * for years and months. */ static npy_uint32 _datetime_factors[] = { 1, /* Years - not used */ 1, /* Months - not used */ 7, /* Weeks -> Days */ - 1, /* Business days - not used */ 24, /* Days -> Hours */ 60, /* Hours -> Minutes */ 60, /* Minutes -> Seconds */ @@ -1637,14 +1538,11 @@ datetime_metadata_divides( /* If the bases are different, factor in a conversion */ if (meta1->base != meta2->base) { /* - * Years, Months, and Business days are incompatible with + * Years and Months are incompatible with * all other units (except years and months are compatible * with each other). */ - if (meta1->base == NPY_FR_B || meta2->base == NPY_FR_B) { - return 0; - } - else if (meta1->base == NPY_FR_Y) { + if (meta1->base == NPY_FR_Y) { if (meta2->base == NPY_FR_M) { num1 *= 12; } @@ -1749,7 +1647,7 @@ compute_datetime_metadata_greatest_common_divisor( } else { /* - * Years, Months, and Business days are incompatible with + * Years and Months are incompatible with * all other units (except years and months are compatible * with each other). */ @@ -1797,27 +1695,6 @@ compute_datetime_metadata_greatest_common_divisor( /* Don't multiply num2 since there is no even factor */ } } - else if (meta1->base == NPY_FR_B || meta2->base == NPY_FR_B) { - if (strict_with_nonlinear_units1 || strict_with_nonlinear_units2) { - goto incompatible_units; - } - else { - if (meta1->base > meta2->base) { - base = meta1->base; - } - else { - base = meta2->base; - } - - /* - * When combining business days with other units, end - * up with days instead of business days. - */ - if (base == NPY_FR_B) { - base = NPY_FR_D; - } - } - } /* Take the greater base (unit sizes are decreasing in enum) */ if (meta1->base > meta2->base) { @@ -2014,8 +1891,6 @@ parse_datetime_unit_from_string(char *str, Py_ssize_t len, char *metastr) return NPY_FR_M; case 'W': return NPY_FR_W; - case 'B': - return NPY_FR_B; case 'D': return NPY_FR_D; case 'h': @@ -3065,7 +2940,6 @@ get_datetime_iso_8601_strlen(int local, NPY_DATETIMEUNIT base) case NPY_FR_h: len += 3; /* "T##" */ case NPY_FR_D: - case NPY_FR_B: case NPY_FR_W: len += 3; /* "-##" */ case NPY_FR_M: @@ -3170,12 +3044,12 @@ make_iso_8601_date(npy_datetimestruct *dts, char *outstr, int outlen, } } /* - * Print business days and weeks with the same precision as days. + * Print weeks with the same precision as days. * * TODO: Could print weeks with YYYY-Www format if the week * epoch is a Monday. */ - else if (base == NPY_FR_B || base == NPY_FR_W) { + else if (base == NPY_FR_W) { base = NPY_FR_D; } @@ -4187,7 +4061,6 @@ convert_timedelta_to_pyobject(npy_timedelta td, PyArray_DatetimeMetaData *meta) if (meta->base > NPY_FR_us || meta->base == NPY_FR_Y || meta->base == NPY_FR_M || - meta->base == NPY_FR_B || meta->base == NPY_FR_GENERIC) { /* Skip use of a tuple for the events, just return the raw int */ return PyLong_FromLongLong(td); diff --git a/numpy/core/src/multiarray/dtype_transfer.c b/numpy/core/src/multiarray/dtype_transfer.c index 8d53a4b653b0..8ab0cdc1bfb3 100644 --- a/numpy/core/src/multiarray/dtype_transfer.c +++ b/numpy/core/src/multiarray/dtype_transfer.c @@ -708,8 +708,9 @@ typedef struct { /* The number of events in the source and destination */ int src_events, dst_events; /* - * The metadata for when dealing with Months, Years, or - * Business Days (all of which behave non-linearly). + * The metadata for when dealing with Months or Years + * which behave non-linearly with respect to the other + * units. */ PyArray_DatetimeMetaData src_meta, dst_meta; } _strided_datetime_cast_data; @@ -889,16 +890,14 @@ get_nbo_cast_datetime_transfer_function(int aligned, /* * Special case the datetime (but not timedelta) with the nonlinear - * units (years, months, business days). For timedelta, an average + * units (years and months). For timedelta, an average * years and months value is used. */ if (src_dtype->type_num == NPY_DATETIME && (src_meta->base == NPY_FR_Y || src_meta->base == NPY_FR_M || - src_meta->base == NPY_FR_B || dst_meta->base == NPY_FR_Y || - dst_meta->base == NPY_FR_M || - dst_meta->base == NPY_FR_B)) { + dst_meta->base == NPY_FR_M)) { memcpy(&data->src_meta, src_meta, sizeof(data->src_meta)); memcpy(&data->dst_meta, dst_meta, sizeof(data->dst_meta)); *out_stransfer = &_strided_to_strided_datetime_general_cast; diff --git a/numpy/core/src/multiarray/scalartypes.c.src b/numpy/core/src/multiarray/scalartypes.c.src index 49ac76441444..380ec3493d7d 100644 --- a/numpy/core/src/multiarray/scalartypes.c.src +++ b/numpy/core/src/multiarray/scalartypes.c.src @@ -691,7 +691,6 @@ static char *_datetime_verbose_strings[] = { "years", "months", "weeks", - "business days", "days", "hours", "minutes", diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index f75b5c1a93dc..320c569f60b7 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -7,7 +7,7 @@ class TestDateTime(TestCase): def test_datetime_dtype_creation(self): - for unit in ['Y', 'M', 'W', 'B', 'D', + for unit in ['Y', 'M', 'W', 'D', 'h', 'm', 's', 'ms', 'us', 'ns', 'ps', 'fs', 'as']: dt1 = np.dtype('M8[750%s]'%unit) @@ -354,7 +354,6 @@ def test_days_to_pydate(self): def test_dtype_comparison(self): assert_(not (np.dtype('M8[us]') == np.dtype('M8[ms]'))) assert_(np.dtype('M8[us]') != np.dtype('M8[ms]')) - assert_(np.dtype('M8[D]') != np.dtype('M8[B]')) assert_(np.dtype('M8[2D]') != np.dtype('M8[D]')) assert_(np.dtype('M8[D]') != np.dtype('M8[2D]')) assert_(np.dtype('M8[Y]//3') != np.dtype('M8[Y]')) @@ -383,7 +382,7 @@ def test_pickle(self): # Check that pickle roundtripping works dt = np.dtype('M8[7D]//3') assert_equal(dt, pickle.loads(pickle.dumps(dt))) - dt = np.dtype('M8[B]') + dt = np.dtype('M8[W]') assert_equal(dt, pickle.loads(pickle.dumps(dt))) def test_dtype_promotion(self): @@ -411,10 +410,6 @@ def test_dtype_promotion(self): # timedelta timedelta raises when there is no reasonable gcd assert_raises(TypeError, np.promote_types, np.dtype('m8[Y]'), np.dtype('m8[D]')) - assert_raises(TypeError, np.promote_types, - np.dtype('m8[Y]'), np.dtype('m8[B]')) - assert_raises(TypeError, np.promote_types, - np.dtype('m8[D]'), np.dtype('m8[B]')) assert_raises(TypeError, np.promote_types, np.dtype('m8[M]'), np.dtype('m8[W]')) # timedelta timedelta may overflow with big unit ranges @@ -835,16 +830,10 @@ def test_divisor_conversion_month(self): assert_(np.dtype('M8[3M/40]') == np.dtype('M8[54h]')) def test_divisor_conversion_week(self): - assert_(np.dtype('m8[W/5]') == np.dtype('m8[B]')) assert_(np.dtype('m8[W/7]') == np.dtype('m8[D]')) assert_(np.dtype('m8[3W/14]') == np.dtype('m8[36h]')) assert_(np.dtype('m8[5W/140]') == np.dtype('m8[360m]')) - def test_divisor_conversion_bday(self): - assert_(np.dtype('M8[B/12]') == np.dtype('M8[2h]')) - assert_(np.dtype('M8[B/120]') == np.dtype('M8[12m]')) - assert_(np.dtype('M8[3B/960]') == np.dtype('M8[270s]')) - def test_divisor_conversion_day(self): assert_(np.dtype('M8[D/12]') == np.dtype('M8[2h]')) assert_(np.dtype('M8[D/120]') == np.dtype('M8[12m]')) From e24e9d4b353247bfbef0f8f92b0b3960ce37349d Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Fri, 10 Jun 2011 10:49:39 -0500 Subject: [PATCH 13/34] ENH: datetime-bday: Add datetime_busday.c/.h, start busday_offset function --- numpy/core/SConscript | 1 + numpy/core/code_generators/genapi.py | 1 + numpy/core/include/numpy/ndarraytypes.h | 26 ++++ numpy/core/setup.py | 1 + numpy/core/src/multiarray/_datetime.h | 6 + numpy/core/src/multiarray/datetime.c | 143 ++++++++++-------- numpy/core/src/multiarray/datetime_busday.c | 116 ++++++++++++++ numpy/core/src/multiarray/datetime_busday.h | 5 + .../src/multiarray/multiarraymodule_onefile.c | 1 + 9 files changed, 233 insertions(+), 67 deletions(-) create mode 100644 numpy/core/src/multiarray/datetime_busday.c create mode 100644 numpy/core/src/multiarray/datetime_busday.h diff --git a/numpy/core/SConscript b/numpy/core/SConscript index bb19b9828e1b..5b25e0b9ca8c 100644 --- a/numpy/core/SConscript +++ b/numpy/core/SConscript @@ -439,6 +439,7 @@ if ENABLE_SEPARATE_COMPILATION: pjoin('src', 'multiarray', 'hashdescr.c'), pjoin('src', 'multiarray', 'arrayobject.c'), pjoin('src', 'multiarray', 'datetime.c'), + pjoin('src', 'multiarray', 'datetime_busday.c'), pjoin('src', 'multiarray', 'numpyos.c'), pjoin('src', 'multiarray', 'flagsobject.c'), pjoin('src', 'multiarray', 'descriptor.c'), diff --git a/numpy/core/code_generators/genapi.py b/numpy/core/code_generators/genapi.py index 3cded28109fa..4b68ba40cb93 100644 --- a/numpy/core/code_generators/genapi.py +++ b/numpy/core/code_generators/genapi.py @@ -44,6 +44,7 @@ join('multiarray', 'conversion_utils.c'), join('multiarray', 'buffer.c'), join('multiarray', 'datetime.c'), + join('multiarray', 'datetime_busday.c'), join('multiarray', 'nditer.c.src'), join('multiarray', 'nditer_pywrap.c'), join('multiarray', 'einsum.c.src'), diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index de6385ded502..d3e28fe93029 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -263,6 +263,32 @@ typedef enum { #define NPY_STR_fs "fs" #define NPY_STR_as "as" +/* + * Business day conventions for mapping invalid business + * days to valid business days. + */ +typedef enum { + /* Go forward in time to the following business day. */ + NPY_BUSDAY_FORWARD, + NPY_BUSDAY_FOLLOWING = NPY_BUSDAY_FORWARD, + /* Go backward in time to the preceding business day. */ + NPY_BUSDAY_BACKWARD, + NPY_BUSDAY_PRECEDING = NPY_BUSDAY_BACKWARD, + /* + * Go forward in time to the following business day, unless it + * crosses a month boundary, in which case go backward + */ + NPY_BUSDAY_MODIFIEDFOLLOWING, + /* + * Go backward in time to the preceding business day, unless it + * crosses a month boundary, in which case go forward. + */ + NPY_BUSDAY_MODIFIEDPRECEDING, + /* Produce a NaT for non-business days. */ + NPY_BUSDAY_NAT, + /* Raise an exception for non-business days. */ + NPY_BUSDAY_RAISE +} NPY_BUSDAY_ROLL; /* * This is to typedef npy_intp to the appropriate pointer size for diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 78bf14f502e1..8e3bff5b5f0c 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -747,6 +747,7 @@ def get_mathlib_info(*args): join('src', 'multiarray', 'numpymemoryview.c'), join('src', 'multiarray', 'buffer.c'), join('src', 'multiarray', 'datetime.c'), + join('src', 'multiarray', 'datetime_busday.c'), join('src', 'multiarray', 'numpyos.c'), join('src', 'multiarray', 'conversion_utils.c'), join('src', 'multiarray', 'flagsobject.c'), diff --git a/numpy/core/src/multiarray/_datetime.h b/numpy/core/src/multiarray/_datetime.h index c2645b34d922..890f80ab601a 100644 --- a/numpy/core/src/multiarray/_datetime.h +++ b/numpy/core/src/multiarray/_datetime.h @@ -4,6 +4,12 @@ NPY_NO_EXPORT void numpy_pydatetime_import(); +/* + * Creates a datetime or timedelta dtype using the provided metadata. + */ +NPY_NO_EXPORT PyArray_Descr * +create_datetime_dtype(int type_num, PyArray_DatetimeMetaData *meta); + /* * This function returns the a new reference to the * capsule with the datetime metadata. diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 881c0f305f98..da48aa405851 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -1,3 +1,12 @@ +/* + * This file implements core functionality for NumPy datetime + * + * Written by Mark Wiebe (mwwiebe@gmail.com) + * Copyright (c) 2011 by Enthought, Inc. + * + * See LICENSE.txt for the license. + */ + #define PY_SSIZE_T_CLEAN #include #include @@ -8,7 +17,6 @@ #include #include "npy_config.h" - #include "numpy/npy_3kcompat.h" #include "numpy/arrayscalars.h" @@ -876,6 +884,73 @@ PyArray_TimedeltaToTimedeltaStruct(npy_timedelta val, NPY_DATETIMEUNIT fr, return; } +/* + * Creates a datetime or timedelta dtype using a copy of the provided metadata. + */ +NPY_NO_EXPORT PyArray_Descr * +create_datetime_dtype(int type_num, PyArray_DatetimeMetaData *meta) +{ + PyArray_Descr *dtype = NULL; + PyArray_DatetimeMetaData *dt_data; + PyObject *metacobj = NULL; + + /* Create a default datetime or timedelta */ + if (type_num == NPY_DATETIME || type_num == NPY_TIMEDELTA) { + dtype = PyArray_DescrNewFromType(type_num); + } + else { + PyErr_SetString(PyExc_RuntimeError, + "Asked to create a datetime type with a non-datetime " + "type number"); + return NULL; + } + + if (dtype == NULL) { + return NULL; + } + + /* + * Remove any reference to old metadata dictionary + * And create a new one for this new dtype + */ + Py_XDECREF(dtype->metadata); + dtype->metadata = PyDict_New(); + if (dtype->metadata == NULL) { + Py_DECREF(dtype); + return NULL; + } + + /* Create a metadata capsule to copy the provided metadata */ + dt_data = PyArray_malloc(sizeof(PyArray_DatetimeMetaData)); + if (dt_data == NULL) { + Py_DECREF(dtype); + PyErr_NoMemory(); + return NULL; + } + + /* Copy the metadata */ + *dt_data = *meta; + + /* Allocate a capsule for it (this claims ownership of dt_data) */ + metacobj = NpyCapsule_FromVoidPtr((void *)dt_data, simple_capsule_dtor); + if (metacobj == NULL) { + Py_DECREF(dtype); + return NULL; + } + + /* Set the metadata object in the dictionary. */ + if (PyDict_SetItemString(dtype->metadata, NPY_METADATA_DTSTR, + metacobj) < 0) { + Py_DECREF(dtype); + Py_DECREF(metacobj); + return NULL; + } + Py_DECREF(metacobj); + + return dtype; +} + + /* * This function returns the a new reference to the * capsule with the datetime metadata. @@ -1082,72 +1157,6 @@ parse_datetime_metadata_from_metastr(char *metastr, Py_ssize_t len, return -1; } -/* - * Creates a datetime or timedelta dtype using the provided metadata. - */ -NPY_NO_EXPORT PyArray_Descr * -create_datetime_dtype(int type_num, PyArray_DatetimeMetaData *meta) -{ - PyArray_Descr *dtype = NULL; - PyArray_DatetimeMetaData *dt_data; - PyObject *metacobj = NULL; - - /* Create a default datetime or timedelta */ - if (type_num == NPY_DATETIME || type_num == NPY_TIMEDELTA) { - dtype = PyArray_DescrNewFromType(type_num); - } - else { - PyErr_SetString(PyExc_RuntimeError, - "Asked to create a datetime type with a non-datetime " - "type number"); - return NULL; - } - - if (dtype == NULL) { - return NULL; - } - - /* - * Remove any reference to old metadata dictionary - * And create a new one for this new dtype - */ - Py_XDECREF(dtype->metadata); - dtype->metadata = PyDict_New(); - if (dtype->metadata == NULL) { - Py_DECREF(dtype); - return NULL; - } - - /* Create a metadata capsule to copy the provided metadata */ - dt_data = PyArray_malloc(sizeof(PyArray_DatetimeMetaData)); - if (dt_data == NULL) { - Py_DECREF(dtype); - PyErr_NoMemory(); - return NULL; - } - - /* Copy the metadata */ - *dt_data = *meta; - - /* Allocate a capsule for it (this claims ownership of dt_data) */ - metacobj = NpyCapsule_FromVoidPtr((void *)dt_data, simple_capsule_dtor); - if (metacobj == NULL) { - Py_DECREF(dtype); - return NULL; - } - - /* Set the metadata object in the dictionary. */ - if (PyDict_SetItemString(dtype->metadata, NPY_METADATA_DTSTR, - metacobj) < 0) { - Py_DECREF(dtype); - Py_DECREF(metacobj); - return NULL; - } - Py_DECREF(metacobj); - - return dtype; -} - /* * Converts a datetype dtype string into a dtype descr object. * The "type" string should be NULL-terminated. diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c new file mode 100644 index 000000000000..8b4d733c5ea4 --- /dev/null +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -0,0 +1,116 @@ +/* + * This file implements business day functionality for NumPy datetime. + * + * Written by Mark Wiebe (mwwiebe@gmail.com) + * Copyright (c) 2011 by Enthought, Inc. + * + * See LICENSE.txt for the license. + */ + +#define PY_SSIZE_T_CLEAN +#include + +#define _MULTIARRAYMODULE +#include + +#include "npy_config.h" +#include "numpy/npy_3kcompat.h" + +#include "numpy/arrayscalars.h" +#include "_datetime.h" +#include "datetime_busday.h" + +/* + * Applies the given offsets in business days to the dates provided. + * This is the low-level function which requires already cleaned input + * data. + * + * dates: An array of dates with 'datetime64[D]' data type. + * offsets: An array safely convertible into type 'timedelta64[D]'. + * out: Either NULL, or an array with 'datetime64[D]' data type + * in which to place the resulting dates. + * roll: A rule for how to treat non-business day dates. + * weekmask: A 7-element boolean mask, 1 for possible business days and 0 + * for non-business days. + * busdays_in_weekmask: A count of how many 1's there are in weekmask. + * holiday_count: The number of elements in 'holidays'. + * holidays: A sorted list of dates matching '[D]' unit metadata, with + * any dates falling on a day of the week without weekmask[i] == 1 + * already filtered out. + * + * For each (date, offset) in the broadcasted pair of (dates, offsets), + * does the following: + * + Applies the 'roll' rule to the date to either produce NaT, raise + * an exception, or land on a valid business day. + * + Adds 'offset' business days to the valid business day found. + * + Sets the value in 'out' if provided, or the allocated output array + * otherwise. + */ +NPY_NO_EXPORT PyArrayObject * +business_day_offset(PyArrayObject *dates, PyArrayObject *offsets, + PyArrayObject *out, + NPY_BUSDAY_ROLL roll, + npy_bool *weekmask, int busdays_in_weekmask, + npy_intp holidays_count, npy_datetime *holidays) +{ + PyArray_DatetimeMetaData temp_meta; + PyArray_Descr *dtypes[3] = {NULL, NULL, NULL}; + + NpyIter *iter = NULL; + PyArrayObject *op[3] = {NULL, NULL, NULL}; + npy_uint32 op_flags[3], flags; + + PyArrayObject *ret = NULL; + + /* First create the data types for dates and offsets */ + temp_meta.base = NPY_FR_D; + temp_meta.num = 1; + temp_meta.events = 1; + dtypes[0] = create_datetime_dtype(NPY_DATETIME, &temp_meta); + if (dtypes[0] == NULL) { + ret = NULL; + goto finish; + } + dtypes[1] = create_datetime_dtype(NPY_DATETIME, &temp_meta); + if (dtypes[1] == NULL) { + ret = NULL; + goto finish; + } + dtypes[2] = dtypes[0]; + Py_INCREF(dtypes[2]); + + /* Set up the iterator parameters */ + flags = NPY_ITER_EXTERNAL_LOOP| + NPY_ITER_BUFFERED| + NPY_ITER_ZEROSIZE_OK; + op[0] = dates; + op_flags[0] = NPY_ITER_READONLY; + op[1] = offsets; + op_flags[1] = NPY_ITER_READONLY; + op[2] = out; + op_flags[2] = NPY_ITER_WRITEONLY | NPY_ITER_ALLOCATE; + + /* Allocate the iterator */ + iter = NpyIter_MultiNew(3, op, flags, NPY_KEEPORDER, NPY_SAFE_CASTING, + op_flags, dtypes); + if (iter == NULL) { + ret = NULL; + goto finish; + } + + /* Get the return object from the iterator */ + ret = NpyIter_GetOperandArray(iter)[2]; + Py_INCREF(ret); + +finish: + Py_XDECREF(dtypes[0]); + Py_XDECREF(dtypes[1]); + Py_XDECREF(dtypes[2]); + if (iter != NULL) { + if (NpyIter_Deallocate(iter) != NPY_SUCCEED) { + Py_XDECREF(ret); + ret = NULL; + } + } + return ret; +} diff --git a/numpy/core/src/multiarray/datetime_busday.h b/numpy/core/src/multiarray/datetime_busday.h new file mode 100644 index 000000000000..bbbb12672138 --- /dev/null +++ b/numpy/core/src/multiarray/datetime_busday.h @@ -0,0 +1,5 @@ +#ifndef _NPY_PRIVATE__DATETIME_BUSDAY_H_ +#define _NPY_PRIVATE__DATETIME_BUSDAY_H_ + + +#endif diff --git a/numpy/core/src/multiarray/multiarraymodule_onefile.c b/numpy/core/src/multiarray/multiarraymodule_onefile.c index 8cf50d9be48f..17c4c739d85e 100644 --- a/numpy/core/src/multiarray/multiarraymodule_onefile.c +++ b/numpy/core/src/multiarray/multiarraymodule_onefile.c @@ -11,6 +11,7 @@ #include "scalarapi.c" #include "datetime.c" +#include "datetime_busday.c" #include "arraytypes.c" #include "hashdescr.c" From 4fced4a2bb556e1915f035604d65e0f83450c775 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Fri, 10 Jun 2011 12:55:14 -0500 Subject: [PATCH 14/34] ENH: datetime-bday: Implement the weekmask part of the busday_offset algorithm --- numpy/core/src/multiarray/_datetime.h | 6 +- numpy/core/src/multiarray/datetime.c | 25 +- numpy/core/src/multiarray/datetime_busday.c | 248 ++++++++++++++++++-- 3 files changed, 261 insertions(+), 18 deletions(-) diff --git a/numpy/core/src/multiarray/_datetime.h b/numpy/core/src/multiarray/_datetime.h index 890f80ab601a..4e6b9d05e501 100644 --- a/numpy/core/src/multiarray/_datetime.h +++ b/numpy/core/src/multiarray/_datetime.h @@ -5,7 +5,7 @@ NPY_NO_EXPORT void numpy_pydatetime_import(); /* - * Creates a datetime or timedelta dtype using the provided metadata. + * Creates a datetime or timedelta dtype using a copy of the provided metadata. */ NPY_NO_EXPORT PyArray_Descr * create_datetime_dtype(int type_num, PyArray_DatetimeMetaData *meta); @@ -41,6 +41,10 @@ convert_datetimestruct_to_datetime(PyArray_DatetimeMetaData *meta, const npy_datetimestruct *dts, npy_datetime *out); +/* Extracts the month number from a 'datetime64[D]' value */ +NPY_NO_EXPORT int +days_to_month_number(npy_datetime days); + /* * Parses the metadata string into the metadata C structure. * diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index da48aa405851..d7c8ab1003e8 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -171,6 +171,29 @@ days_to_yearsdays(npy_int64 *days_) return year + 2000; } +/* Extracts the month number from a 'datetime64[D]' value */ +NPY_NO_EXPORT int +days_to_month_number(npy_datetime days) +{ + npy_int64 year; + int *month_lengths, i; + + year = days_to_yearsdays(&days); + month_lengths = days_in_month[is_leapyear(year)]; + + for (i = 0; i < 12; ++i) { + if (days < month_lengths[i]) { + return i + 1; + } + else { + days -= month_lengths[i]; + } + } + + /* Should never get here */ + return 1; +} + /* * Fills in the year, month, day in 'dts' based on the days * offset from 1970. @@ -181,8 +204,8 @@ set_datetimestruct_days(npy_int64 days, npy_datetimestruct *dts) int *month_lengths, i; dts->year = days_to_yearsdays(&days); - month_lengths = days_in_month[is_leapyear(dts->year)]; + for (i = 0; i < 12; ++i) { if (days < month_lengths[i]) { dts->month = i + 1; diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index 8b4d733c5ea4..56d9ce9cb271 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -20,23 +20,196 @@ #include "_datetime.h" #include "datetime_busday.h" + +/* + * Applies the 'roll' strategy to 'date', placing the result in 'out' + * and setting 'out_day_of_week' to the day of the week that results. + * + * Returns 0 on success, -1 on failure. + */ +static int +apply_business_day_roll(npy_datetime date, npy_datetime *out, + int *out_day_of_week, + NPY_BUSDAY_ROLL roll, + npy_bool *weekmask, + npy_datetime *holidays_begin, npy_datetime *holidays_end) +{ + int day_of_week; + + /* Deal with NaT input */ + if (date == NPY_DATETIME_NAT) { + *out = NPY_DATETIME_NAT; + if (roll == NPY_BUSDAY_RAISE) { + PyErr_SetString(PyExc_ValueError, + "NaT input in busday_offset"); + return -1; + } + else { + return 0; + } + } + + /* Get the day of the week for 'date' (1970-01-05 is Monday) */ + day_of_week = (int)((date - 4) % 7); + if (day_of_week < 0) { + day_of_week += 7; + } + + /* Apply the 'roll' if it's not a business day */ + if (weekmask[day_of_week] == 0) { + npy_datetime start_date = date; + int start_day_of_week = day_of_week; + + switch (roll) { + case NPY_BUSDAY_FOLLOWING: + case NPY_BUSDAY_MODIFIEDFOLLOWING: { + do { + ++date; + if (++day_of_week == 7) { + day_of_week = 0; + } + } while (weekmask[day_of_week] == 0); + + if (roll == NPY_BUSDAY_MODIFIEDFOLLOWING) { + /* If we crossed a month boundary, do preceding instead */ + if (days_to_month_number(start_date) != + days_to_month_number(date)) { + date = start_date; + day_of_week = start_day_of_week; + + do { + --date; + if (--day_of_week == -1) { + day_of_week = 6; + } + } while (weekmask[day_of_week] == 0); + } + } + break; + } + case NPY_BUSDAY_PRECEDING: + case NPY_BUSDAY_MODIFIEDPRECEDING: { + do { + --date; + if (--day_of_week == -1) { + day_of_week = 6; + } + } while (weekmask[day_of_week] == 0); + + if (roll == NPY_BUSDAY_MODIFIEDPRECEDING) { + /* If we crossed a month boundary, do following instead */ + if (days_to_month_number(start_date) != + days_to_month_number(date)) { + date = start_date; + day_of_week = start_day_of_week; + + do { + ++date; + if (++day_of_week == 7) { + day_of_week = 0; + } + } while (weekmask[day_of_week] == 0); + } + } + break; + } + case NPY_BUSDAY_NAT: { + date = NPY_DATETIME_NAT; + break; + } + case NPY_BUSDAY_RAISE: { + *out = NPY_DATETIME_NAT; + PyErr_SetString(PyExc_ValueError, + "Non-business day date in busday_offset"); + return -1; + } + } + } + + *out = date; + *out_day_of_week = day_of_week; + + return 0; +} + +/* + * Applies a single business day offset. See the function + * business_day_offset for the meaning of all the parameters. + * + * Returns 0 on success, -1 on failure. + */ +static int +apply_business_day_offset(npy_datetime date, npy_int64 offset, + npy_datetime *out, + NPY_BUSDAY_ROLL roll, + npy_bool *weekmask, int busdays_in_weekmask, + npy_datetime *holidays_begin, npy_datetime *holidays_end) +{ + int day_of_week = 0; + + /* Roll the date to a business day */ + if (apply_business_day_roll(date, &date, &day_of_week, + roll, + weekmask, + holidays_begin, holidays_end) < 0) { + return -1; + } + + /* If we get a NaT, just return it */ + if (date == NPY_DATETIME_NAT) { + return 0; + } + + /* Now we're on a valid business day */ + if (offset > 0) { + /* Jump by as many weeks as we can */ + date += (offset / busdays_in_weekmask) * 7; + offset = offset % busdays_in_weekmask; + + /* Step until we use up the rest of the offset */ + while (offset > 0) { + ++date; + if (++day_of_week == 7) { + day_of_week = 0; + } + offset -= weekmask[day_of_week]; + } + } + else if (offset < 0) { + /* Jump by as many weeks as we can */ + date += (offset / busdays_in_weekmask) * 7; + offset = offset % busdays_in_weekmask; + + /* Step until we use up the rest of the offset */ + while (offset < 0) { + --date; + if (--day_of_week == -1) { + day_of_week = 6; + } + offset += weekmask[day_of_week]; + } + } + + *out = date; + return 0; +} + /* * Applies the given offsets in business days to the dates provided. * This is the low-level function which requires already cleaned input * data. * * dates: An array of dates with 'datetime64[D]' data type. - * offsets: An array safely convertible into type 'timedelta64[D]'. + * offsets: An array safely convertible into type int64. * out: Either NULL, or an array with 'datetime64[D]' data type * in which to place the resulting dates. * roll: A rule for how to treat non-business day dates. * weekmask: A 7-element boolean mask, 1 for possible business days and 0 * for non-business days. * busdays_in_weekmask: A count of how many 1's there are in weekmask. - * holiday_count: The number of elements in 'holidays'. - * holidays: A sorted list of dates matching '[D]' unit metadata, with - * any dates falling on a day of the week without weekmask[i] == 1 - * already filtered out. + * holidays_begin/holidays_end: A sorted list of dates matching '[D]' + * unit metadata, with any dates falling on a day of the + * week without weekmask[i] == 1 already filtered out. * * For each (date, offset) in the broadcasted pair of (dates, offsets), * does the following: @@ -51,7 +224,7 @@ business_day_offset(PyArrayObject *dates, PyArrayObject *offsets, PyArrayObject *out, NPY_BUSDAY_ROLL roll, npy_bool *weekmask, int busdays_in_weekmask, - npy_intp holidays_count, npy_datetime *holidays) + npy_datetime *holidays_begin, npy_datetime *holidays_end) { PyArray_DatetimeMetaData temp_meta; PyArray_Descr *dtypes[3] = {NULL, NULL, NULL}; @@ -68,13 +241,11 @@ business_day_offset(PyArrayObject *dates, PyArrayObject *offsets, temp_meta.events = 1; dtypes[0] = create_datetime_dtype(NPY_DATETIME, &temp_meta); if (dtypes[0] == NULL) { - ret = NULL; - goto finish; + goto fail; } - dtypes[1] = create_datetime_dtype(NPY_DATETIME, &temp_meta); + dtypes[1] = PyArray_DescrFromType(NPY_INT64); if (dtypes[1] == NULL) { - ret = NULL; - goto finish; + goto fail; } dtypes[2] = dtypes[0]; Py_INCREF(dtypes[2]); @@ -84,24 +255,69 @@ business_day_offset(PyArrayObject *dates, PyArrayObject *offsets, NPY_ITER_BUFFERED| NPY_ITER_ZEROSIZE_OK; op[0] = dates; - op_flags[0] = NPY_ITER_READONLY; + op_flags[0] = NPY_ITER_READONLY | NPY_ITER_ALIGNED; op[1] = offsets; - op_flags[1] = NPY_ITER_READONLY; + op_flags[1] = NPY_ITER_READONLY | NPY_ITER_ALIGNED; op[2] = out; - op_flags[2] = NPY_ITER_WRITEONLY | NPY_ITER_ALLOCATE; + op_flags[2] = NPY_ITER_WRITEONLY | NPY_ITER_ALLOCATE | NPY_ITER_ALIGNED; /* Allocate the iterator */ iter = NpyIter_MultiNew(3, op, flags, NPY_KEEPORDER, NPY_SAFE_CASTING, op_flags, dtypes); if (iter == NULL) { - ret = NULL; - goto finish; + goto fail; + } + + /* Loop over all elements */ + if (NpyIter_GetIterSize(iter) > 0) { + NpyIter_IterNextFunc *iternext; + char **dataptr; + npy_intp *strideptr, *innersizeptr; + + iternext = NpyIter_GetIterNext(iter, NULL); + if (iternext == NULL) { + goto fail; + } + dataptr = NpyIter_GetDataPtrArray(iter); + strideptr = NpyIter_GetInnerStrideArray(iter); + innersizeptr = NpyIter_GetInnerLoopSizePtr(iter); + + do { + char *data_dates = dataptr[0]; + char *data_offsets = dataptr[1]; + char *data_out = dataptr[2]; + npy_intp stride_dates = strideptr[0]; + npy_intp stride_offsets = strideptr[1]; + npy_intp stride_out = strideptr[2]; + npy_intp count = *innersizeptr; + + while (count--) { + if (apply_business_day_offset(*(npy_int64 *)data_dates, + *(npy_int64 *)data_offsets, + (npy_int64 *)data_out, + roll, + weekmask, busdays_in_weekmask, + holidays_begin, holidays_end) < 0) { + goto fail; + } + + data_dates += stride_dates; + data_offsets += stride_offsets; + data_out += stride_out; + } + } while (iternext(iter)); } /* Get the return object from the iterator */ ret = NpyIter_GetOperandArray(iter)[2]; Py_INCREF(ret); + goto finish; + +fail: + Py_XDECREF(ret); + ret = NULL; + finish: Py_XDECREF(dtypes[0]); Py_XDECREF(dtypes[1]); From 8a8a84a89fe45187bbfb284f3a0ebdbd52396d41 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Fri, 10 Jun 2011 14:36:51 -0500 Subject: [PATCH 15/34] ENH: datetime-bday: Connect busday_offset so it can be called from Python --- numpy/core/numerictypes.py | 5 +- numpy/core/src/multiarray/datetime_busday.c | 374 +++++++++++++++++++ numpy/core/src/multiarray/datetime_busday.h | 7 + numpy/core/src/multiarray/multiarraymodule.c | 6 + numpy/core/src/multiarray/nditer.c.src | 39 +- 5 files changed, 419 insertions(+), 12 deletions(-) diff --git a/numpy/core/numerictypes.py b/numpy/core/numerictypes.py index 6d655cb1707e..f287e1774c1e 100644 --- a/numpy/core/numerictypes.py +++ b/numpy/core/numerictypes.py @@ -92,10 +92,11 @@ __all__ = ['sctypeDict', 'sctypeNA', 'typeDict', 'typeNA', 'sctypes', 'ScalarType', 'obj2sctype', 'cast', 'nbytes', 'sctype2char', 'maximum_sctype', 'issctype', 'typecodes', 'find_common_type', - 'issubdtype','datetime_data','datetime_as_string'] + 'issubdtype','datetime_data','datetime_as_string', 'busday_offset'] from numpy.core.multiarray import typeinfo, ndarray, array, \ - empty, dtype, datetime_data, datetime_as_string + empty, dtype, datetime_data, datetime_as_string, \ + busday_offset import types as _types import sys diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index 56d9ce9cb271..9a2272612eed 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -235,6 +235,20 @@ business_day_offset(PyArrayObject *dates, PyArrayObject *offsets, PyArrayObject *ret = NULL; + if (busdays_in_weekmask == 0) { + PyErr_SetString(PyExc_ValueError, + "the business day weekmask must have at least one " + "valid business day"); + return NULL; + } + + int i; + printf("weekmask: "); + for (i = 0; i < 7; ++i) { + printf("%d", weekmask[i]); + } + printf("\n"); + /* First create the data types for dates and offsets */ temp_meta.base = NPY_FR_D; temp_meta.num = 1; @@ -330,3 +344,363 @@ business_day_offset(PyArrayObject *dates, PyArrayObject *offsets, } return ret; } + +static int +PyArray_BusDayRollConverter(PyObject *roll_in, NPY_BUSDAY_ROLL *roll) +{ + PyObject *obj = roll_in; + char *str; + Py_ssize_t len; + + /* Make obj into an ASCII string */ + Py_INCREF(obj); + if (PyUnicode_Check(obj)) { + /* accept unicode input */ + PyObject *obj_str; + obj_str = PyUnicode_AsASCIIString(obj); + if (obj_str == NULL) { + Py_DECREF(obj); + return 0; + } + Py_DECREF(obj); + obj = obj_str; + } + + if (PyBytes_AsStringAndSize(obj, &str, &len) < 0) { + Py_DECREF(obj); + return 0; + } + + /* Use switch statements to quickly isolate the right enum value */ + switch (str[0]) { + case 'b': + if (strcmp(str, "backward") == 0) { + *roll = NPY_BUSDAY_BACKWARD; + goto finish; + } + break; + case 'f': + if (len > 2) switch (str[2]) { + case 'r': + if (strcmp(str, "forward") == 0) { + *roll = NPY_BUSDAY_FORWARD; + goto finish; + } + break; + case 'l': + if (strcmp(str, "following") == 0) { + *roll = NPY_BUSDAY_FOLLOWING; + goto finish; + } + break; + } + break; + case 'm': + if (len > 8) switch (str[8]) { + case 'f': + if (strcmp(str, "modifiedfollowing") == 0) { + *roll = NPY_BUSDAY_MODIFIEDFOLLOWING; + goto finish; + } + break; + case 'p': + if (strcmp(str, "modifiedpreceding") == 0) { + *roll = NPY_BUSDAY_MODIFIEDFOLLOWING; + goto finish; + } + break; + } + break; + case 'n': + if (strcmp(str, "nat") == 0) { + *roll = NPY_BUSDAY_NAT; + goto finish; + } + break; + case 'p': + if (strcmp(str, "preceding") == 0) { + *roll = NPY_BUSDAY_PRECEDING; + goto finish; + } + break; + case 'r': + if (strcmp(str, "raise") == 0) { + *roll = NPY_BUSDAY_RAISE; + goto finish; + } + break; + } + + PyErr_Format(PyExc_ValueError, + "Invalid business day roll parameter \"%s\"", + str); + Py_DECREF(obj); + return 0; + +finish: + Py_DECREF(obj); + return 1; +} + +static int +PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask) +{ + PyObject *obj = weekmask_in; + + printf("getting weekmask\n"); + + /* Make obj into an ASCII string if it is UNICODE */ + Py_INCREF(obj); + if (PyUnicode_Check(obj)) { + /* accept unicode input */ + PyObject *obj_str; + obj_str = PyUnicode_AsASCIIString(obj); + if (obj_str == NULL) { + Py_DECREF(obj); + return 0; + } + Py_DECREF(obj); + obj = obj_str; + } + + if (PyBytes_Check(obj)) { + char *str; + Py_ssize_t len; + + if (PyBytes_AsStringAndSize(obj, &str, &len) < 0) { + Py_DECREF(obj); + return 0; + } + + /* Length 7 is a string like "1111100" */ + if (len == 7) { + int i; + for (i = 0; i < 7; ++i) { + switch(str[i]) { + case '0': + weekmask[i] = 0; + break; + case '1': + weekmask[i] = 1; + break; + default: + goto invalid_weekmask_string; + } + } + + goto finish; + } + /* Length divisible by 3 is a string like "Mon" or "MonWedFri" */ + else if (len % 3 == 0) { + int i; + memset(weekmask, 0, 7); + for (i = 0; i < len; i += 3) { + switch (str[i]) { + case 'M': + if (str[i+1] == 'o' && str[i+2] == 'n') { + weekmask[0] = 1; + } + else { + goto invalid_weekmask_string; + } + break; + case 'T': + if (str[i+1] == 'u' && str[i+2] == 'e') { + weekmask[1] = 1; + } + else if (str[i+1] == 'h' && str[i+2] == 'u') { + weekmask[3] = 1; + } + else { + goto invalid_weekmask_string; + } + break; + case 'W': + if (str[i+1] == 'e' && str[i+2] == 'd') { + weekmask[2] = 1; + } + else { + goto invalid_weekmask_string; + } + break; + case 'F': + if (str[i+1] == 'r' && str[i+2] == 'i') { + weekmask[4] = 1; + } + else { + goto invalid_weekmask_string; + } + break; + case 'S': + if (str[i+1] == 'a' && str[i+2] == 't') { + weekmask[5] = 1; + } + else if (str[i+1] == 'u' && str[i+2] == 'n') { + weekmask[6] = 1; + } + else { + goto invalid_weekmask_string; + } + break; + } + } + + goto finish; + } + +invalid_weekmask_string: + PyErr_Format(PyExc_ValueError, + "Invalid business day weekmask string \"%s\"", + str); + Py_DECREF(obj); + return 0; + } + /* Something like [1,1,1,1,1,0,0] */ + else if (PySequence_Check(obj)) { + if (PySequence_Size(obj) != 7 || + (PyArray_Check(obj) && PyArray_NDIM(obj) != 1)) { + PyErr_SetString(PyExc_ValueError, + "A business day weekmask array must have length 7"); + Py_DECREF(obj); + return 0; + } + else { + int i; + PyObject *f; + + for (i = 0; i < 7; ++i) { + long val; + + f = PySequence_GetItem(obj, i); + if (f == NULL) { + Py_DECREF(obj); + return 0; + } + + val = PyInt_AsLong(f); + if (val == -1 && PyErr_Occurred()) { + Py_DECREF(obj); + return 0; + } + if (val == 0) { + weekmask[i] = 0; + } + else if (val == 1) { + weekmask[i] = 1; + } + else { + PyErr_SetString(PyExc_ValueError, + "A business day weekmask array must have all " + "1's and 0's"); + Py_DECREF(obj); + return 0; + } + } + + goto finish; + } + } + + PyErr_SetString(PyExc_ValueError, + "Couldn't convert object into a business day weekmask"); + Py_DECREF(obj); + return 0; + + +finish: + Py_DECREF(obj); + return 1; +} + +/* + * This is the 'busday_offset' function exposed for calling + * from Python. + */ +NPY_NO_EXPORT PyObject * +array_busday_offset(PyObject *NPY_UNUSED(self), + PyObject *args, PyObject *kwds) +{ + char *kwlist[] = {"dates", "offsets", "roll", + "weekmask", "holidays", NULL}; + + PyObject *dates_in = NULL, *offsets_in = NULL, + *holidays_in = NULL, *busdaydef_in = NULL, + *out_in = NULL; + + PyArrayObject *dates = NULL, *offsets = NULL, *out = NULL, *ret; + NPY_BUSDAY_ROLL roll = NPY_BUSDAY_RAISE; + npy_bool weekmask[7] = {1, 1, 1, 1, 1, 0, 0}; + int i, busdays_in_weekmask; + npy_datetime *holidays_begin = NULL, *holidays_end = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OO|O&O&OOO:busday_offset", kwlist, + &dates_in, + &offsets_in, + &PyArray_BusDayRollConverter, &roll, + &PyArray_WeekMaskConverter, &weekmask[0], + &holidays_in, + &busdaydef_in, + &out_in)) { + return NULL; + } + + /* Make 'dates' into an array */ + if (PyArray_Check(dates_in)) { + dates = (PyArrayObject *)dates_in; + Py_INCREF(dates); + } + else { + PyArray_Descr *date_dtype; + + /* Use the datetime dtype with generic units so it fills it in */ + date_dtype = PyArray_DescrFromType(NPY_DATETIME); + if (date_dtype == NULL) { + goto fail; + } + + /* This steals the date_dtype reference */ + dates = (PyArrayObject *)PyArray_FromAny(dates_in, date_dtype, + 0, 0, 0, dates_in); + if (dates == NULL) { + goto fail; + } + } + + /* Make 'offsets' into an array */ + offsets = (PyArrayObject *)PyArray_FromAny(offsets_in, + PyArray_DescrFromType(NPY_INT64), + 0, 0, 0, offsets_in); + if (offsets == NULL) { + goto fail; + } + + /* Make sure 'out' is an array if it's provided */ + if (out_in != NULL) { + if (!PyArray_Check(out_in)) { + PyErr_SetString(PyExc_ValueError, + "busday_offset: must provide a NumPy array for 'out'"); + goto fail; + } + out = (PyArrayObject *)out_in; + } + + busdays_in_weekmask = 0; + for (i = 0; i < 7; ++i) { + busdays_in_weekmask += weekmask[i]; + } + + ret = business_day_offset(dates, offsets, out, roll, + weekmask, busdays_in_weekmask, + holidays_begin, holidays_end); + + Py_DECREF(dates); + Py_DECREF(offsets); + + return out == NULL ? PyArray_Return(ret) : (PyObject *)ret; +fail: + Py_XDECREF(dates); + Py_XDECREF(offsets); + + return NULL; +} diff --git a/numpy/core/src/multiarray/datetime_busday.h b/numpy/core/src/multiarray/datetime_busday.h index bbbb12672138..aef050463517 100644 --- a/numpy/core/src/multiarray/datetime_busday.h +++ b/numpy/core/src/multiarray/datetime_busday.h @@ -1,5 +1,12 @@ #ifndef _NPY_PRIVATE__DATETIME_BUSDAY_H_ #define _NPY_PRIVATE__DATETIME_BUSDAY_H_ +/* + * This is the 'busday_offset' function exposed for calling + * from Python. + */ +NPY_NO_EXPORT PyObject * +array_busday_offset(PyObject *NPY_UNUSED(self), + PyObject *args, PyObject *kwds); #endif diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index f1103ef7f97e..fb3b0c352764 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -45,6 +45,7 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0; #include "convert_datatype.h" #include "nditer_pywrap.h" #include "_datetime.h" +#include "datetime_busday.h" /* Only here for API compatibility */ NPY_NO_EXPORT PyTypeObject PyBigArray_Type; @@ -3590,12 +3591,17 @@ static struct PyMethodDef array_module_methods[] = { {"result_type", (PyCFunction)array_result_type, METH_VARARGS, NULL}, + /* Datetime-related functions */ {"datetime_data", (PyCFunction)array_datetime_data, METH_VARARGS, NULL}, {"datetime_as_string", (PyCFunction)array_datetime_as_string, METH_VARARGS | METH_KEYWORDS, NULL}, + /* Datetime business-day API */ + {"busday_offset", + (PyCFunction)array_busday_offset, + METH_VARARGS | METH_KEYWORDS, NULL}, #if !defined(NPY_PY3K) {"newbuffer", (PyCFunction)new_buffer, diff --git a/numpy/core/src/multiarray/nditer.c.src b/numpy/core/src/multiarray/nditer.c.src index 676d9496918c..3d7be4137d43 100644 --- a/numpy/core/src/multiarray/nditer.c.src +++ b/numpy/core/src/multiarray/nditer.c.src @@ -3342,11 +3342,20 @@ npyiter_check_casting(int nop, PyArrayObject **op, !PyArray_CanCastArrayTo(op[iop], op_dtype[iop], casting)) { - PyErr_Format(PyExc_TypeError, - "Iterator operand %d dtype could not be cast " - "to the requested dtype, according to " - "the casting rule given, %s", (int)iop, - npyiter_casting_to_string(casting)); + PyObject *errmsg; + errmsg = PyUString_FromFormat( + "Iterator operand %d dtype could not be cast from ", + (int)iop); + PyUString_ConcatAndDel(&errmsg, + PyObject_Repr((PyObject *)PyArray_DESCR(op[iop]))); + PyUString_ConcatAndDel(&errmsg, + PyUString_FromString(" to ")); + PyUString_ConcatAndDel(&errmsg, + PyObject_Repr((PyObject *)op_dtype[iop])); + PyUString_ConcatAndDel(&errmsg, + PyUString_FromFormat(" according to the rule %s", + npyiter_casting_to_string(casting))); + PyErr_SetObject(PyExc_TypeError, errmsg); return 0; } /* Check write (temp -> op) casting */ @@ -3354,11 +3363,21 @@ npyiter_check_casting(int nop, PyArrayObject **op, !PyArray_CanCastTypeTo(op_dtype[iop], PyArray_DESCR(op[iop]), casting)) { - PyErr_Format(PyExc_TypeError, - "Iterator requested dtype could not be cast " - "to the operand %d dtype, according to " - "the casting rule given, %s", (int)iop, - npyiter_casting_to_string(casting)); + PyObject *errmsg; + errmsg = PyUString_FromString( + "Iterator requested dtype could not be cast from "); + PyUString_ConcatAndDel(&errmsg, + PyObject_Repr((PyObject *)op_dtype[iop])); + PyUString_ConcatAndDel(&errmsg, + PyUString_FromString(" to ")); + PyUString_ConcatAndDel(&errmsg, + PyObject_Repr((PyObject *)PyArray_DESCR(op[iop]))); + PyUString_ConcatAndDel(&errmsg, + PyUString_FromFormat(", the operand %d dtype, " + "according to the rule %s", + (int)iop, + npyiter_casting_to_string(casting))); + PyErr_SetObject(PyExc_TypeError, errmsg); return 0; } From db62c35f9f9e9b3fe3d8c6463cab34967024dbf2 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Fri, 10 Jun 2011 17:09:39 -0500 Subject: [PATCH 16/34] ENH: datetime-autounit: Unit detection working with arrays, fix ufunc reductions The default NPY_DATETIME type was still in microseconds, because it wasn't using the NPY_DATETIME_DEFAULTUNIT macro as it should have been. The reduction functions in ufuncs didn't respect the metadata appropriately. --- numpy/core/arrayprint.py | 5 +- numpy/core/src/multiarray/_datetime.h | 8 + numpy/core/src/multiarray/arraytypes.c.src | 2 +- numpy/core/src/multiarray/ctors.c | 21 ++ numpy/core/src/multiarray/datetime.c | 283 ++++++++++++++++++++ numpy/core/src/multiarray/datetime_busday.c | 9 - numpy/core/src/umath/ufunc_object.c | 40 ++- numpy/core/tests/test_datetime.py | 17 +- 8 files changed, 364 insertions(+), 21 deletions(-) diff --git a/numpy/core/arrayprint.py b/numpy/core/arrayprint.py index 8595d6f021a9..556d4da04b22 100644 --- a/numpy/core/arrayprint.py +++ b/numpy/core/arrayprint.py @@ -718,8 +718,5 @@ def __init__(self, data): self.format = '%' + str(max_str_len) + 'd' def __call__(self, x): - if _MININT < x < _MAXINT: - return self.format % x.astype('i8') - else: - return "%s" % x + return self.format % x.astype('i8') diff --git a/numpy/core/src/multiarray/_datetime.h b/numpy/core/src/multiarray/_datetime.h index 4e6b9d05e501..53cb53fdfac9 100644 --- a/numpy/core/src/multiarray/_datetime.h +++ b/numpy/core/src/multiarray/_datetime.h @@ -390,4 +390,12 @@ NPY_NO_EXPORT PyArrayObject * datetime_arange(PyObject *start, PyObject *stop, PyObject *step, PyArray_Descr *dtype); +/* + * Examines all the objects in the given Python object by + * recursively descending the sequence structure. Returns a + * datetime or timedelta type with metadata based on the data. + */ +NPY_NO_EXPORT PyArray_Descr * +find_object_datetime_type(PyObject *obj, int type_num); + #endif diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index e55c7c1337ae..f5f173d75500 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -3434,7 +3434,7 @@ _init_datetime_descr(PyArray_Descr *descr) PyObject *cobj; dt_data = _pya_malloc(sizeof(PyArray_DatetimeMetaData)); - dt_data->base = NPY_FR_us; + dt_data->base = NPY_DATETIME_DEFAULTUNIT; dt_data->num = 1; dt_data->events = 1; diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 35ed706c4b4c..0ae7434819ce 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1690,6 +1690,27 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, } } } + /* Treat datetime generic units with the same idea as flexible strings */ + else if (newtype != NULL && newtype->type_num == NPY_DATETIME) { + PyArray_DatetimeMetaData *meta = + get_datetime_metadata_from_dtype(newtype); + if (meta == NULL) { + Py_DECREF(newtype); + return NULL; + } + + if (meta->base == NPY_FR_GENERIC) { + /* Detect the unit from the input's data */ + PyArray_Descr *dtype = find_object_datetime_type(op, + newtype->type_num); + if (dtype == NULL) { + Py_DECREF(newtype); + return NULL; + } + Py_DECREF(newtype); + newtype = dtype; + } + } /* If we got dimensions and dtype instead of an array */ if (arr == NULL) { diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index d7c8ab1003e8..ab8142da6d37 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -4685,3 +4685,286 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, return ret; } + + +/* + * Recursively determines the metadata for an NPY_DATETIME dtype. + * + * Returns 0 on success, -1 on failure. + */ +static int +recursive_find_object_datetime64_type(PyObject *obj, + PyArray_DatetimeMetaData *meta) +{ + /* Array -> use its metadata */ + if (PyArray_Check(obj)) { + PyArray_Descr *obj_dtype = PyArray_DESCR(obj); + /* If the array has metadata, use it */ + if (obj_dtype->type_num == NPY_DATETIME || + obj_dtype->type_num == NPY_TIMEDELTA) { + PyArray_DatetimeMetaData *tmp_meta; + + /* Get the metadata from the type */ + tmp_meta = get_datetime_metadata_from_dtype(obj_dtype); + if (tmp_meta == NULL) { + return -1; + } + + /* Combine it with 'meta' */ + if (compute_datetime_metadata_greatest_common_divisor(meta, + tmp_meta, meta, 0, 0) < 0) { + return -1; + } + + return 0; + } + /* If it's not an object array, stop looking */ + else if (obj_dtype->type_num != NPY_OBJECT) { + return 0; + } + } + /* Datetime scalar -> use its metadata */ + else if (PyArray_IsScalar(obj, Datetime)) { + PyDatetimeScalarObject *dts = (PyDatetimeScalarObject *)obj; + + /* Combine it with 'meta' */ + if (compute_datetime_metadata_greatest_common_divisor(meta, + &dts->obmeta, meta, 0, 0) < 0) { + return -1; + } + + return 0; + } + /* String -> parse it to find out */ + else if (PyBytes_Check(obj) || PyUnicode_Check(obj)) { + npy_datetime tmp = 0; + PyArray_DatetimeMetaData tmp_meta; + + tmp_meta.base = -1; + tmp_meta.num = 1; + tmp_meta.events = 1; + + if (convert_pyobject_to_datetime(&tmp_meta, obj, &tmp) < 0) { + /* If it's a value error, clear the error */ + if (PyErr_Occurred() && + PyErr_GivenExceptionMatches(PyErr_Occurred(), + PyExc_ValueError)) { + PyErr_Clear(); + return 0; + } + /* Otherwise propagate the error */ + else { + return -1; + } + } + + /* Combine it with 'meta' */ + if (compute_datetime_metadata_greatest_common_divisor(meta, + &tmp_meta, meta, 0, 0) < 0) { + return -1; + } + + return 0; + } + /* Python date object -> 'D' */ + else if (PyDate_Check(obj)) { + PyArray_DatetimeMetaData tmp_meta; + + tmp_meta.base = NPY_FR_D; + tmp_meta.num = 1; + tmp_meta.events = 1; + + /* Combine it with 'meta' */ + if (compute_datetime_metadata_greatest_common_divisor(meta, + &tmp_meta, meta, 0, 0) < 0) { + return -1; + } + + return 0; + } + /* Python datetime object -> 'us' */ + else if (PyDateTime_Check(obj)) { + PyArray_DatetimeMetaData tmp_meta; + + tmp_meta.base = NPY_FR_us; + tmp_meta.num = 1; + tmp_meta.events = 1; + + /* Combine it with 'meta' */ + if (compute_datetime_metadata_greatest_common_divisor(meta, + &tmp_meta, meta, 0, 0) < 0) { + return -1; + } + + return 0; + } + + /* Now check if what we have left is a sequence for recursion */ + if (PySequence_Check(obj)) { + Py_ssize_t i, len = PySequence_Size(obj); + if (len < 0 && PyErr_Occurred()) { + return -1; + } + + for (i = 0; i < len; ++i) { + PyObject *f = PySequence_GetItem(obj, i); + if (f == NULL) { + return -1; + } + if (f == obj) { + Py_DECREF(f); + return 0; + } + if (recursive_find_object_datetime64_type(f, meta) < 0) { + Py_DECREF(f); + return -1; + } + Py_DECREF(f); + } + + return 0; + } + /* Otherwise ignore it */ + else { + return 0; + } +} + +/* + * Recursively determines the metadata for an NPY_TIMEDELTA dtype. + * + * Returns 0 on success, -1 on failure. + */ +static int +recursive_find_object_timedelta64_type(PyObject *obj, + PyArray_DatetimeMetaData *meta) +{ + /* Array -> use its metadata */ + if (PyArray_Check(obj)) { + PyArray_Descr *obj_dtype = PyArray_DESCR(obj); + /* If the array has metadata, use it */ + if (obj_dtype->type_num == NPY_DATETIME || + obj_dtype->type_num == NPY_TIMEDELTA) { + PyArray_DatetimeMetaData *tmp_meta; + + /* Get the metadata from the type */ + tmp_meta = get_datetime_metadata_from_dtype(obj_dtype); + if (tmp_meta == NULL) { + return -1; + } + + /* Combine it with 'meta' */ + if (compute_datetime_metadata_greatest_common_divisor(meta, + tmp_meta, meta, 0, 0) < 0) { + return -1; + } + + return 0; + } + /* If it's not an object array, stop looking */ + else if (obj_dtype->type_num != NPY_OBJECT) { + return 0; + } + } + /* Datetime scalar -> use its metadata */ + else if (PyArray_IsScalar(obj, Timedelta)) { + PyTimedeltaScalarObject *dts = (PyTimedeltaScalarObject *)obj; + + /* Combine it with 'meta' */ + if (compute_datetime_metadata_greatest_common_divisor(meta, + &dts->obmeta, meta, 1, 1) < 0) { + return -1; + } + + return 0; + } + /* String -> parse it to find out */ + else if (PyBytes_Check(obj) || PyUnicode_Check(obj)) { + /* No timedelta parser yet */ + return 0; + } + /* Python timedelta object -> 'us' */ + else if (PyDelta_Check(obj)) { + PyArray_DatetimeMetaData tmp_meta; + + tmp_meta.base = NPY_FR_us; + tmp_meta.num = 1; + tmp_meta.events = 1; + + /* Combine it with 'meta' */ + if (compute_datetime_metadata_greatest_common_divisor(meta, + &tmp_meta, meta, 0, 0) < 0) { + return -1; + } + + return 0; + } + + /* Now check if what we have left is a sequence for recursion */ + if (PySequence_Check(obj)) { + Py_ssize_t i, len = PySequence_Size(obj); + if (len < 0 && PyErr_Occurred()) { + return -1; + } + + for (i = 0; i < len; ++i) { + PyObject *f = PySequence_GetItem(obj, i); + if (f == NULL) { + return -1; + } + if (f == obj) { + Py_DECREF(f); + return 0; + } + if (recursive_find_object_timedelta64_type(f, meta) < 0) { + Py_DECREF(f); + return -1; + } + Py_DECREF(f); + } + + return 0; + } + /* Otherwise ignore it */ + else { + return 0; + } +} + +/* + * Examines all the objects in the given Python object by + * recursively descending the sequence structure. Returns a + * datetime or timedelta type with metadata based on the data. + */ +NPY_NO_EXPORT PyArray_Descr * +find_object_datetime_type(PyObject *obj, int type_num) +{ + PyArray_DatetimeMetaData meta; + + meta.base = NPY_FR_GENERIC; + meta.num = 1; + meta.events = 1; + + if (type_num == NPY_DATETIME) { + if (recursive_find_object_datetime64_type(obj, &meta) < 0) { + return NULL; + } + else { + return create_datetime_dtype(type_num, &meta); + } + } + else if (type_num == NPY_TIMEDELTA) { + if (recursive_find_object_timedelta64_type(obj, &meta) < 0) { + return NULL; + } + else { + return create_datetime_dtype(type_num, &meta); + } + } + else { + PyErr_SetString(PyExc_ValueError, + "find_object_datetime_type needs a datetime or " + "timedelta type number"); + return NULL; + } +} diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index 9a2272612eed..92eba9f1cf75 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -242,13 +242,6 @@ business_day_offset(PyArrayObject *dates, PyArrayObject *offsets, return NULL; } - int i; - printf("weekmask: "); - for (i = 0; i < 7; ++i) { - printf("%d", weekmask[i]); - } - printf("\n"); - /* First create the data types for dates and offsets */ temp_meta.base = NPY_FR_D; temp_meta.num = 1; @@ -447,8 +440,6 @@ PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask) { PyObject *obj = weekmask_in; - printf("getting weekmask\n"); - /* Make obj into an ASCII string if it is UNICODE */ Py_INCREF(obj); if (PyUnicode_Check(obj)) { diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index e37dd0167046..c166cb1b5b6c 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -4126,8 +4126,24 @@ PyUFunc_ReductionOp(PyUFuncObject *self, PyArrayObject *arr, ndim = PyArray_NDIM(arr); - /* Set up the output data type */ - op_dtypes[0] = PyArray_DescrFromType(otype_final); + /* + * Set up the output data type, using the input's exact + * data type if the type number didn't change to preserve + * metadata + */ + if (PyArray_DESCR(arr)->type_num == otype_final) { + if (PyArray_ISNBO(PyArray_DESCR(arr)->byteorder)) { + op_dtypes[0] = PyArray_DESCR(arr); + Py_INCREF(op_dtypes[0]); + } + else { + op_dtypes[0] = PyArray_DescrNewByteorder(PyArray_DESCR(arr), + NPY_NATIVE); + } + } + else { + op_dtypes[0] = PyArray_DescrFromType(otype_final); + } if (op_dtypes[0] == NULL) { goto fail; } @@ -4777,8 +4793,24 @@ PyUFunc_Reduceat(PyUFuncObject *self, PyArrayObject *arr, PyArrayObject *ind, ndim = PyArray_NDIM(arr); - /* Set up the output data type */ - op_dtypes[0] = PyArray_DescrFromType(otype_final); + /* + * Set up the output data type, using the input's exact + * data type if the type number didn't change to preserve + * metadata + */ + if (PyArray_DESCR(arr)->type_num == otype_final) { + if (PyArray_ISNBO(PyArray_DESCR(arr)->byteorder)) { + op_dtypes[0] = PyArray_DESCR(arr); + Py_INCREF(op_dtypes[0]); + } + else { + op_dtypes[0] = PyArray_DescrNewByteorder(PyArray_DESCR(arr), + NPY_NATIVE); + } + } + else { + op_dtypes[0] = PyArray_DescrFromType(otype_final); + } if (op_dtypes[0] == NULL) { goto fail; } diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index 320c569f60b7..af2eaf69f5cd 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -667,9 +667,9 @@ def test_datetime_subtract(self): assert_equal((dtc - tdb).dtype, np.dtype('M8[h]')) # M8 - M8 with different goes to higher precision - assert_equal(dtc - dtd, 0) + assert_equal(dtc - dtd, np.timedelta64(0,'h')) assert_equal((dtc - dtd).dtype, np.dtype('m8[h]')) - assert_equal(dtd - dtc, 0) + assert_equal(dtd - dtc, np.timedelta64(0,'h')) assert_equal((dtd - dtc).dtype, np.dtype('m8[h]')) # m8 - M8 @@ -1148,11 +1148,22 @@ def test_timedelta_arange(self): assert_raises(TypeError, np.arange, np.timedelta64(0,'Y'), np.timedelta64(5,'D')) + def test_maximum_reduce(self): + a = np.array(['2010-01-02','1999-03-14','1833-03'], dtype='M8[D]') + assert_equal(np.maximum.reduce(a).dtype, np.dtype('M8[D]')) + assert_equal(np.maximum.reduce(a), + np.datetime64('2010-01-02')) + + a = np.array([1,4,0,7,2], dtype='m8[s]') + assert_equal(np.maximum.reduce(a).dtype, np.dtype('m8[s]')) + assert_equal(np.maximum.reduce(a), + np.timedelta64(7, 's')) + class TestDateTimeData(TestCase): def test_basic(self): a = np.array(['1980-03-23'], dtype=np.datetime64) - assert_equal(np.datetime_data(a.dtype), (asbytes('us'), 1, 1)) + assert_equal(np.datetime_data(a.dtype), (asbytes('D'), 1, 1)) if __name__ == "__main__": run_module_suite() From 29a3cebde0cb0c18e4120701e12e244a261a710f Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Fri, 10 Jun 2011 17:45:17 -0500 Subject: [PATCH 17/34] TST: datetime-bday: Write some tests for busday_offset --- numpy/core/tests/test_datetime.py | 49 ++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index af2eaf69f5cd..371a29c8eec0 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -1148,7 +1148,7 @@ def test_timedelta_arange(self): assert_raises(TypeError, np.arange, np.timedelta64(0,'Y'), np.timedelta64(5,'D')) - def test_maximum_reduce(self): + def test_datetime_maximum_reduce(self): a = np.array(['2010-01-02','1999-03-14','1833-03'], dtype='M8[D]') assert_equal(np.maximum.reduce(a).dtype, np.dtype('M8[D]')) assert_equal(np.maximum.reduce(a), @@ -1159,6 +1159,53 @@ def test_maximum_reduce(self): assert_equal(np.maximum.reduce(a), np.timedelta64(7, 's')) + def test_datetime_busday_offset(self): + # First Monday in June + assert_equal( + np.busday_offset('2011-06',0,roll='forward',weekmask='Mon'), + np.datetime64('2011-06-06')) + # Last Monday in June + assert_equal( + np.busday_offset('2011-07',-1,roll='forward',weekmask='Mon'), + np.datetime64('2011-06-27')) + assert_equal( + np.busday_offset('2011-07',-1,roll='forward',weekmask='Mon'), + np.datetime64('2011-06-27')) + + # Default M-F business days, different roll modes + assert_equal(np.busday_offset('2010-08', 0, roll='backward'), + np.datetime64('2010-07-30')) + assert_equal(np.busday_offset('2010-08', 0, roll='preceding'), + np.datetime64('2010-07-30')) + assert_equal(np.busday_offset('2010-08', 0, roll='modifiedpreceding'), + np.datetime64('2010-08-02')) + assert_equal(np.busday_offset('2010-08', 0, roll='modifiedfollowing'), + np.datetime64('2010-08-02')) + assert_equal(np.busday_offset('2010-08', 0, roll='forward'), + np.datetime64('2010-08-02')) + assert_equal(np.busday_offset('2010-08', 0, roll='following'), + np.datetime64('2010-08-02')) + assert_equal(np.busday_offset('2010-10-30', 0, roll='following'), + np.datetime64('2010-11-01')) + assert_equal( + np.busday_offset('2010-10-30', 0, roll='modifiedfollowing'), + np.datetime64('2010-10-29')) + assert_equal( + np.busday_offset('2010-10-30', 0, roll='modifiedpreceding'), + np.datetime64('2010-10-29')) + # roll='raise' by default + assert_raises(ValueError, np.busday_offset, '2011-06-04', 0) + + # Bigger offset values + assert_equal(np.busday_offset('2006-02-01', 25), + np.datetime64('2006-03-08')) + assert_equal(np.busday_offset('2006-03-08', -25), + np.datetime64('2006-02-01')) + assert_equal(np.busday_offset('2007-02-25', 11, weekmask='SatSun'), + np.datetime64('2007-04-07')) + assert_equal(np.busday_offset('2007-04-07', -11, weekmask='SatSun'), + np.datetime64('2007-02-25')) + class TestDateTimeData(TestCase): def test_basic(self): From 3d8fda55fdc88af22e87300a4f0723dbc19ef3ff Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Mon, 13 Jun 2011 11:23:20 -0500 Subject: [PATCH 18/34] ENH: datetime-bday: Functions to get and normalize a list of holidays --- numpy/core/src/multiarray/_datetime.h | 6 + numpy/core/src/multiarray/datetime.c | 12 ++ numpy/core/src/multiarray/datetime_busday.c | 171 +++++++++++++++++- numpy/core/src/multiarray/dtype_transfer.c | 49 +++++ .../core/src/private/lowlevel_strided_loops.h | 13 ++ 5 files changed, 245 insertions(+), 6 deletions(-) diff --git a/numpy/core/src/multiarray/_datetime.h b/numpy/core/src/multiarray/_datetime.h index 53cb53fdfac9..12e833378e15 100644 --- a/numpy/core/src/multiarray/_datetime.h +++ b/numpy/core/src/multiarray/_datetime.h @@ -10,6 +10,12 @@ numpy_pydatetime_import(); NPY_NO_EXPORT PyArray_Descr * create_datetime_dtype(int type_num, PyArray_DatetimeMetaData *meta); +/* + * Creates a datetime or timedelta dtype using the given unit. + */ +NPY_NO_EXPORT PyArray_Descr * +create_datetime_dtype_with_unit(int type_num, NPY_DATETIMEUNIT unit); + /* * This function returns the a new reference to the * capsule with the datetime metadata. diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index ab8142da6d37..a9f2f6749eb8 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -973,6 +973,18 @@ create_datetime_dtype(int type_num, PyArray_DatetimeMetaData *meta) return dtype; } +/* + * Creates a datetime or timedelta dtype using the given unit. + */ +NPY_NO_EXPORT PyArray_Descr * +create_datetime_dtype_with_unit(int type_num, NPY_DATETIMEUNIT unit) +{ + PyArray_DatetimeMetaData meta; + meta.base = unit; + meta.num = 1; + meta.events = 1; + return create_datetime_dtype(type_num, &meta); +} /* * This function returns the a new reference to the diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index 92eba9f1cf75..977c0ca41d9c 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -17,6 +17,7 @@ #include "numpy/npy_3kcompat.h" #include "numpy/arrayscalars.h" +#include "lowlevel_strided_loops.h" #include "_datetime.h" #include "datetime_busday.h" @@ -603,6 +604,155 @@ PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask) return 1; } +static int +qsort_datetime_compare(const void *elem1, const void *elem2) +{ + npy_datetime e1 = *(npy_datetime *)elem1; + npy_datetime e2 = *(npy_datetime *)elem2; + + return (e1 < e2) ? -1 : (e1 == e2) ? 0 : 1; +} + +/* + * A list of holidays, which should sorted, not contain any + * duplicates or NaTs, and not include any days already excluded + * by the associated weekmask. + * + * The data is manually managed with PyArray_malloc/PyArray_free. + */ +typedef struct { + npy_datetime *begin, *end; +} npy_holidayslist; + +/* + * Sorts the the array of dates provided in place and removes + * NaT, duplicates and any date which is already excluded on account + * of the weekmask. + * + * Returns the number of dates left after removing weekmask-excluded + * dates. + */ +static void +normalize_holidays_list(npy_holidayslist *holidays, npy_bool *weekmask) +{ + npy_datetime *dates = holidays->begin; + npy_intp count = holidays->end - dates; + + npy_datetime lastdate = NPY_DATETIME_NAT; + npy_intp trimcount, i; + int day_of_week; + + /* Sort the dates */ + qsort(dates, count, sizeof(npy_datetime), &qsort_datetime_compare); + + /* Sweep throught the array, eliminating unnecessary values */ + trimcount = 0; + for (i = 0; i < count; ++i) { + npy_datetime date = dates[i]; + + /* Skip any NaT or duplicate */ + if (date != NPY_DATETIME_NAT && date != lastdate) { + /* Get the day of the week (1970-01-05 is Monday) */ + day_of_week = (int)((date - 4) % 7); + if (day_of_week < 0) { + day_of_week += 7; + } + + /* + * If the holiday falls on a possible business day, + * then keep it. + */ + if (weekmask[day_of_week] == 1) { + dates[trimcount++] = date; + lastdate = date; + } + } + } + + /* Adjust the end of the holidays array */ + holidays->end = dates + trimcount; +} + +/* + * Converts a Python input into a non-normalized list of holidays. + * + * IMPORTANT: This function can't do the normalization, because it doesn't + * know the weekmask. You must call 'normalize_holiday_list' + * on the result before using it. + */ +static int +PyArray_HolidaysConverter(PyObject *dates_in, npy_holidayslist *holidays) +{ + PyArrayObject *dates = NULL; + PyArray_Descr *date_dtype = NULL; + npy_intp count; + + /* Make 'dates' into an array */ + if (PyArray_Check(dates_in)) { + dates = (PyArrayObject *)dates_in; + Py_INCREF(dates); + } + else { + PyArray_Descr *datetime_dtype; + + /* Use the datetime dtype with generic units so it fills it in */ + datetime_dtype = PyArray_DescrFromType(NPY_DATETIME); + if (datetime_dtype == NULL) { + goto fail; + } + + /* This steals the datetime_dtype reference */ + dates = (PyArrayObject *)PyArray_FromAny(dates_in, datetime_dtype, + 0, 0, 0, dates_in); + if (dates == NULL) { + goto fail; + } + } + + date_dtype = create_datetime_dtype_with_unit(NPY_DATETIME, NPY_FR_D); + if (date_dtype == NULL) { + goto fail; + } + + if (!PyArray_CanCastTypeTo(PyArray_DESCR(dates), + date_dtype, NPY_SAFE_CASTING)) { + PyErr_SetString(PyExc_ValueError, "Cannot safely convert " + "provided holidays input into an array of dates"); + goto fail; + } + if (PyArray_NDIM(dates) != 1) { + PyErr_SetString(PyExc_ValueError, "holidays must be a provided " + "as a one-dimensional array"); + goto fail; + } + + /* Allocate the memory for the dates */ + count = PyArray_DIM(dates, 0); + holidays->begin = PyArray_malloc(sizeof(npy_datetime) * count); + if (holidays->begin == NULL) { + PyErr_NoMemory(); + goto fail; + } + holidays->end = holidays->begin + count; + + /* Cast the data into a raw date array */ + if (PyArray_CastRawArrays(count, + PyArray_BYTES(dates), (char *)holidays->begin, + PyArray_STRIDE(dates, 0), sizeof(npy_datetime), + PyArray_DESCR(dates), date_dtype, + 0) != NPY_SUCCEED) { + goto fail; + } + + Py_DECREF(date_dtype); + date_dtype = NULL; + +fail: + Py_XDECREF(dates); + Py_XDECREF(date_dtype); + return 0; +} + /* * This is the 'busday_offset' function exposed for calling * from Python. @@ -615,22 +765,21 @@ array_busday_offset(PyObject *NPY_UNUSED(self), "weekmask", "holidays", NULL}; PyObject *dates_in = NULL, *offsets_in = NULL, - *holidays_in = NULL, *busdaydef_in = NULL, - *out_in = NULL; + *busdaydef_in = NULL, *out_in = NULL; PyArrayObject *dates = NULL, *offsets = NULL, *out = NULL, *ret; NPY_BUSDAY_ROLL roll = NPY_BUSDAY_RAISE; npy_bool weekmask[7] = {1, 1, 1, 1, 1, 0, 0}; int i, busdays_in_weekmask; - npy_datetime *holidays_begin = NULL, *holidays_end = NULL; + npy_holidayslist holidays = {NULL, NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O&O&OOO:busday_offset", kwlist, + "OO|O&O&O&OO:busday_offset", kwlist, &dates_in, &offsets_in, &PyArray_BusDayRollConverter, &roll, &PyArray_WeekMaskConverter, &weekmask[0], - &holidays_in, + &PyArray_HolidaysConverter, &holidays, &busdaydef_in, &out_in)) { return NULL; @@ -676,22 +825,32 @@ array_busday_offset(PyObject *NPY_UNUSED(self), out = (PyArrayObject *)out_in; } + /* Count the number of business days in a week */ busdays_in_weekmask = 0; for (i = 0; i < 7; ++i) { busdays_in_weekmask += weekmask[i]; } + /* The holidays list must be normalized before using it */ + normalize_holidays_list(&holidays, weekmask); + ret = business_day_offset(dates, offsets, out, roll, weekmask, busdays_in_weekmask, - holidays_begin, holidays_end); + holidays.begin, holidays.end); Py_DECREF(dates); Py_DECREF(offsets); + if (holidays.begin != NULL) { + PyArray_free(holidays.begin); + } return out == NULL ? PyArray_Return(ret) : (PyObject *)ret; fail: Py_XDECREF(dates); Py_XDECREF(offsets); + if (holidays.begin != NULL) { + PyArray_free(holidays.begin); + } return NULL; } diff --git a/numpy/core/src/multiarray/dtype_transfer.c b/numpy/core/src/multiarray/dtype_transfer.c index 8ab0cdc1bfb3..fa5573ad54f6 100644 --- a/numpy/core/src/multiarray/dtype_transfer.c +++ b/numpy/core/src/multiarray/dtype_transfer.c @@ -3234,3 +3234,52 @@ PyArray_GetDTypeTransferFunction(int aligned, out_stransfer, out_transferdata, out_needs_api); } + +NPY_NO_EXPORT int +PyArray_CastRawArrays(npy_intp count, + char *src, char *dst, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + int move_references) +{ + PyArray_StridedTransferFn *stransfer = NULL; + void *transferdata = NULL; + int aligned = 1, needs_api = 0; + + /* Make sure the copy is reasonable */ + if (dst_stride == 0 && count > 1) { + PyErr_SetString(PyExc_ValueError, + "NumPy CastRawArrays cannot do a reduction"); + return NPY_FAIL; + } + else if (count == 0) { + return NPY_SUCCEED; + } + + /* Check data alignment */ + aligned = (((npy_intp)src_dtype | src_stride) & + (src_dtype->alignment - 1)) == 0 && + (((npy_intp)dst_dtype | dst_stride) & + (dst_dtype->alignment - 1)) == 0; + + /* Get the function to do the casting */ + if (PyArray_GetDTypeTransferFunction(aligned, + src_stride, dst_stride, + src_dtype, dst_dtype, + move_references, + &stransfer, &transferdata, + &needs_api) != NPY_SUCCEED) { + return NPY_FAIL; + } + + /* Cast */ + stransfer(dst, dst_stride, src, src_stride, count, + src_dtype->elsize, transferdata); + + /* Cleanup */ + PyArray_FreeStridedTransferData(transferdata); + + /* If needs_api was set to 1, it may have raised a Python exception */ + return (needs_api && PyErr_Occurred()) ? NPY_FAIL : NPY_SUCCEED; +} + diff --git a/numpy/core/src/private/lowlevel_strided_loops.h b/numpy/core/src/private/lowlevel_strided_loops.h index 5fc42bc40616..79b8f6fd7a20 100644 --- a/numpy/core/src/private/lowlevel_strided_loops.h +++ b/numpy/core/src/private/lowlevel_strided_loops.h @@ -173,6 +173,19 @@ PyArray_GetDTypeTransferFunction(int aligned, void **out_transferdata, int *out_needs_api); +/* + * Casts the specified number of elements from 'src' with data type + * 'src_dtype' to 'dst' with 'dst_dtype'. See + * PyArray_GetDTypeTransferFunction for more details. + * + * returns NPY_SUCCEED or NPY_FAIL. + */ +NPY_NO_EXPORT int +PyArray_CastRawArrays(npy_intp count, + char *src, char *dst, + npy_intp src_stride, npy_intp dst_stride, + PyArray_Descr *src_dtype, PyArray_Descr *dst_dtype, + int move_references); /* * These two functions copy or convert the data of an n-dimensional array * to/from a 1-dimensional strided buffer. These functions will only call From b90d182e958ca8ab75f33d013c3a500340ac56ae Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Mon, 13 Jun 2011 16:21:16 -0500 Subject: [PATCH 19/34] ENH: datetime-bday: Get holidays working with busday_offset --- numpy/core/src/multiarray/datetime_busday.c | 155 +++++++++++++++++-- numpy/core/tests/test_datetime.py | 156 ++++++++++++++++++++ 2 files changed, 302 insertions(+), 9 deletions(-) diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index 977c0ca41d9c..02618a9afda9 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -21,7 +21,110 @@ #include "_datetime.h" #include "datetime_busday.h" +/* + * Returns 1 if the date is a holiday (contained in the sorted + * list of dates), 0 otherwise. + * + * The holidays list should be normalized. + */ +static int +is_holiday(npy_datetime date, + npy_datetime *holidays_begin, npy_datetime *holidays_end) +{ + npy_datetime *trial; + + /* Simple binary search */ + while (holidays_begin < holidays_end) { + trial = holidays_begin + (holidays_end - holidays_begin) / 2; + + if (date < *trial) { + holidays_end = trial; + } + else if (date > *trial) { + holidays_begin = trial + 1; + } + else { + return 1; + } + } + + /* Not found */ + return 0; +} +/* + * Finds the earliest holiday which is on or after 'date'. If 'date' does not + * appear within the holiday range, returns 'holidays_begin' if 'date' + * is before all holidays, or 'holidays_end' if 'date' is after all + * holidays. + * + * To remove all the holidays before 'date' from a holiday range, do: + * + * holidays_begin = find_holiday_earliest_on_or_after(date, + * holidays_begin, holidays_end); + * + * The holidays list should be normalized. + */ +static npy_datetime * +find_earliest_holiday_on_or_after(npy_datetime date, + npy_datetime *holidays_begin, npy_datetime *holidays_end) +{ + npy_datetime *trial; + + /* Simple binary search */ + while (holidays_begin < holidays_end) { + trial = holidays_begin + (holidays_end - holidays_begin) / 2; + + if (date < *trial) { + holidays_end = trial; + } + else if (date > *trial) { + holidays_begin = trial + 1; + } + else { + return trial; + } + } + + return holidays_begin; +} + +/* + * Finds the earliest holiday which is after 'date'. If 'date' does not + * appear within the holiday range, returns 'holidays_begin' if 'date' + * is before all holidays, or 'holidays_end' if 'date' is after all + * holidays. + * + * To remove all the holidays after 'date' from a holiday range, do: + * + * holidays_end = find_holiday_earliest_after(date, + * holidays_begin, holidays_end); + * + * The holidays list should be normalized. + */ +static npy_datetime * +find_earliest_holiday_after(npy_datetime date, + npy_datetime *holidays_begin, npy_datetime *holidays_end) +{ + npy_datetime *trial; + + /* Simple binary search */ + while (holidays_begin < holidays_end) { + trial = holidays_begin + (holidays_end - holidays_begin) / 2; + + if (date < *trial) { + holidays_end = trial; + } + else if (date > *trial) { + holidays_begin = trial + 1; + } + else { + return trial + 1; + } + } + + return holidays_begin; +} /* * Applies the 'roll' strategy to 'date', placing the result in 'out' * and setting 'out_day_of_week' to the day of the week that results. @@ -57,7 +160,8 @@ apply_business_day_roll(npy_datetime date, npy_datetime *out, } /* Apply the 'roll' if it's not a business day */ - if (weekmask[day_of_week] == 0) { + if (weekmask[day_of_week] == 0 || + is_holiday(date, holidays_begin, holidays_end)) { npy_datetime start_date = date; int start_day_of_week = day_of_week; @@ -69,7 +173,8 @@ apply_business_day_roll(npy_datetime date, npy_datetime *out, if (++day_of_week == 7) { day_of_week = 0; } - } while (weekmask[day_of_week] == 0); + } while (weekmask[day_of_week] == 0 || + is_holiday(date, holidays_begin, holidays_end)); if (roll == NPY_BUSDAY_MODIFIEDFOLLOWING) { /* If we crossed a month boundary, do preceding instead */ @@ -83,7 +188,8 @@ apply_business_day_roll(npy_datetime date, npy_datetime *out, if (--day_of_week == -1) { day_of_week = 6; } - } while (weekmask[day_of_week] == 0); + } while (weekmask[day_of_week] == 0 || + is_holiday(date, holidays_begin, holidays_end)); } } break; @@ -95,7 +201,8 @@ apply_business_day_roll(npy_datetime date, npy_datetime *out, if (--day_of_week == -1) { day_of_week = 6; } - } while (weekmask[day_of_week] == 0); + } while (weekmask[day_of_week] == 0 || + is_holiday(date, holidays_begin, holidays_end)); if (roll == NPY_BUSDAY_MODIFIEDPRECEDING) { /* If we crossed a month boundary, do following instead */ @@ -109,7 +216,8 @@ apply_business_day_roll(npy_datetime date, npy_datetime *out, if (++day_of_week == 7) { day_of_week = 0; } - } while (weekmask[day_of_week] == 0); + } while (weekmask[day_of_week] == 0 || + is_holiday(date, holidays_begin, holidays_end)); } } break; @@ -147,6 +255,7 @@ apply_business_day_offset(npy_datetime date, npy_int64 offset, npy_datetime *holidays_begin, npy_datetime *holidays_end) { int day_of_week = 0; + npy_datetime *holidays_temp; /* Roll the date to a business day */ if (apply_business_day_roll(date, &date, &day_of_week, @@ -163,31 +272,57 @@ apply_business_day_offset(npy_datetime date, npy_int64 offset, /* Now we're on a valid business day */ if (offset > 0) { + /* Remove any earlier holidays */ + holidays_begin = find_earliest_holiday_on_or_after(date, + holidays_begin, holidays_end); + /* Jump by as many weeks as we can */ date += (offset / busdays_in_weekmask) * 7; offset = offset % busdays_in_weekmask; + /* Adjust based on the number of holidays we crossed */ + holidays_temp = find_earliest_holiday_after(date, + holidays_begin, holidays_end); + offset += holidays_temp - holidays_begin; + holidays_begin = holidays_temp; + /* Step until we use up the rest of the offset */ while (offset > 0) { ++date; if (++day_of_week == 7) { day_of_week = 0; } - offset -= weekmask[day_of_week]; + if (weekmask[day_of_week] && !is_holiday(date, + holidays_begin, holidays_end)) { + offset--; + } } } else if (offset < 0) { + /* Remove any later holidays */ + holidays_end = find_earliest_holiday_after(date, + holidays_begin, holidays_end); + /* Jump by as many weeks as we can */ date += (offset / busdays_in_weekmask) * 7; offset = offset % busdays_in_weekmask; + /* Adjust based on the number of holidays we crossed */ + holidays_temp = find_earliest_holiday_on_or_after(date, + holidays_begin, holidays_end); + offset -= holidays_end - holidays_temp; + holidays_end = holidays_temp; + /* Step until we use up the rest of the offset */ while (offset < 0) { --date; if (--day_of_week == -1) { day_of_week = 6; } - offset += weekmask[day_of_week]; + if (weekmask[day_of_week] && !is_holiday(date, + holidays_begin, holidays_end)) { + offset++; + } } } @@ -744,8 +879,10 @@ PyArray_HolidaysConverter(PyObject *dates_in, npy_holidayslist *holidays) goto fail; } + Py_DECREF(dates); Py_DECREF(date_dtype); - date_dtype = NULL; + + return 1; fail: Py_XDECREF(dates); @@ -762,7 +899,7 @@ array_busday_offset(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) { char *kwlist[] = {"dates", "offsets", "roll", - "weekmask", "holidays", NULL}; + "weekmask", "holidays", "busdaydef", "out", NULL}; PyObject *dates_in = NULL, *offsets_in = NULL, *busdaydef_in = NULL, *out_in = NULL; diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index 371a29c8eec0..9d80200b59b0 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -1206,6 +1206,162 @@ def test_datetime_busday_offset(self): assert_equal(np.busday_offset('2007-04-07', -11, weekmask='SatSun'), np.datetime64('2007-02-25')) + def test_datetime_busday_holidays_offset(self): + # With exactly one holiday + assert_equal( + np.busday_offset('2011-11-10', 1, holidays=['2011-11-11']), + np.datetime64('2011-11-14')) + assert_equal( + np.busday_offset('2011-11-04', 5, holidays=['2011-11-11']), + np.datetime64('2011-11-14')) + assert_equal( + np.busday_offset('2011-11-10', 5, holidays=['2011-11-11']), + np.datetime64('2011-11-18')) + assert_equal( + np.busday_offset('2011-11-14', -1, holidays=['2011-11-11']), + np.datetime64('2011-11-10')) + assert_equal( + np.busday_offset('2011-11-18', -5, holidays=['2011-11-11']), + np.datetime64('2011-11-10')) + assert_equal( + np.busday_offset('2011-11-14', -5, holidays=['2011-11-11']), + np.datetime64('2011-11-04')) + # With the holiday appearing twice + assert_equal( + np.busday_offset('2011-11-10', 1, + holidays=['2011-11-11', '2011-11-11']), + np.datetime64('2011-11-14')) + assert_equal( + np.busday_offset('2011-11-14', -1, + holidays=['2011-11-11', '2011-11-11']), + np.datetime64('2011-11-10')) + # With a NaT holiday + assert_equal( + np.busday_offset('2011-11-10', 1, + holidays=['2011-11-11', 'NaT']), + np.datetime64('2011-11-14')) + assert_equal( + np.busday_offset('2011-11-14', -1, + holidays=['NaT', '2011-11-11']), + np.datetime64('2011-11-10')) + # With another holiday after + assert_equal( + np.busday_offset('2011-11-10', 1, + holidays=['2011-11-11', '2011-11-24']), + np.datetime64('2011-11-14')) + assert_equal( + np.busday_offset('2011-11-14', -1, + holidays=['2011-11-11', '2011-11-24']), + np.datetime64('2011-11-10')) + # With another holiday before + assert_equal( + np.busday_offset('2011-11-10', 1, + holidays=['2011-10-10', '2011-11-11']), + np.datetime64('2011-11-14')) + assert_equal( + np.busday_offset('2011-11-14', -1, + holidays=['2011-10-10', '2011-11-11']), + np.datetime64('2011-11-10')) + # With another holiday before and after + assert_equal( + np.busday_offset('2011-11-10', 1, + holidays=['2011-10-10', '2011-11-11', '2011-11-24']), + np.datetime64('2011-11-14')) + assert_equal( + np.busday_offset('2011-11-14', -1, + holidays=['2011-10-10', '2011-11-11', '2011-11-24']), + np.datetime64('2011-11-10')) + + # A bigger forward jump across more than one week/holiday + holidays=['2011-10-10', '2011-11-11', '2011-11-24', + '2011-12-25', '2011-05-30', '2011-02-21', + '2011-12-26', '2012-01-02'] + assert_equal( + np.busday_offset('2011-10-03', 4, holidays=holidays), + np.busday_offset('2011-10-03', 4)) + assert_equal( + np.busday_offset('2011-10-03', 5, holidays=holidays), + np.busday_offset('2011-10-03', 5 + 1)) + assert_equal( + np.busday_offset('2011-10-03', 27, holidays=holidays), + np.busday_offset('2011-10-03', 27 + 1)) + assert_equal( + np.busday_offset('2011-10-03', 28, holidays=holidays), + np.busday_offset('2011-10-03', 28 + 2)) + assert_equal( + np.busday_offset('2011-10-03', 35, holidays=holidays), + np.busday_offset('2011-10-03', 35 + 2)) + assert_equal( + np.busday_offset('2011-10-03', 36, holidays=holidays), + np.busday_offset('2011-10-03', 36 + 3)) + assert_equal( + np.busday_offset('2011-10-03', 56, holidays=holidays), + np.busday_offset('2011-10-03', 56 + 3)) + assert_equal( + np.busday_offset('2011-10-03', 57, holidays=holidays), + np.busday_offset('2011-10-03', 57 + 4)) + assert_equal( + np.busday_offset('2011-10-03', 60, holidays=holidays), + np.busday_offset('2011-10-03', 60 + 4)) + assert_equal( + np.busday_offset('2011-10-03', 61, holidays=holidays), + np.busday_offset('2011-10-03', 61 + 5)) + # A bigger backward jump across more than one week/holiday + assert_equal( + np.busday_offset('2012-01-03', -1, holidays=holidays), + np.busday_offset('2012-01-03', -1 - 1)) + assert_equal( + np.busday_offset('2012-01-03', -4, holidays=holidays), + np.busday_offset('2012-01-03', -4 - 1)) + assert_equal( + np.busday_offset('2012-01-03', -5, holidays=holidays), + np.busday_offset('2012-01-03', -5 - 2)) + assert_equal( + np.busday_offset('2012-01-03', -25, holidays=holidays), + np.busday_offset('2012-01-03', -25 - 2)) + assert_equal( + np.busday_offset('2012-01-03', -26, holidays=holidays), + np.busday_offset('2012-01-03', -26 - 3)) + assert_equal( + np.busday_offset('2012-01-03', -33, holidays=holidays), + np.busday_offset('2012-01-03', -33 - 3)) + assert_equal( + np.busday_offset('2012-01-03', -34, holidays=holidays), + np.busday_offset('2012-01-03', -34 - 4)) + assert_equal( + np.busday_offset('2012-01-03', -56, holidays=holidays), + np.busday_offset('2012-01-03', -56 - 4)) + assert_equal( + np.busday_offset('2012-01-03', -57, holidays=holidays), + np.busday_offset('2012-01-03', -57 - 5)) + + # Roll with the holidays + assert_equal( + np.busday_offset('2011-12-25', 0, + roll='forward', holidays=holidays), + np.datetime64('2011-12-27')) + assert_equal( + np.busday_offset('2011-12-26', 0, + roll='forward', holidays=holidays), + np.datetime64('2011-12-27')) + assert_equal( + np.busday_offset('2011-12-26', 0, + roll='backward', holidays=holidays), + np.datetime64('2011-12-23')) + assert_equal( + np.busday_offset('2012-02-27', 0, + roll='modifiedfollowing', + holidays=['2012-02-27', '2012-02-26', '2012-02-28', + '2012-03-01', '2012-02-29']), + np.datetime64('2012-02-24')) + assert_equal( + np.busday_offset('2012-03-06', 0, + roll='modifiedpreceding', + holidays=['2012-03-02', '2012-03-03', '2012-03-01', + '2012-03-05', '2012-03-07', '2012-03-06']), + np.datetime64('2012-03-08')) + + class TestDateTimeData(TestCase): def test_basic(self): From 68f1cf2216b0b5f61eb900b50e30496c0567b8dd Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Mon, 13 Jun 2011 18:11:20 -0500 Subject: [PATCH 20/34] ENH: datetime-bday: Create the np.busdaydef business day definition object --- numpy/core/SConscript | 1 + numpy/core/code_generators/genapi.py | 1 + numpy/core/numerictypes.py | 5 +- numpy/core/setup.py | 1 + numpy/core/src/multiarray/datetime_busday.c | 71 +++--- numpy/core/src/multiarray/datetime_busday.h | 41 ++++ .../core/src/multiarray/datetime_busdaydef.c | 217 ++++++++++++++++++ .../core/src/multiarray/datetime_busdaydef.h | 15 ++ numpy/core/src/multiarray/multiarraymodule.c | 9 + .../src/multiarray/multiarraymodule_onefile.c | 1 + numpy/core/src/multiarray/nditer_pywrap.c | 2 +- numpy/core/tests/test_datetime.py | 24 ++ 12 files changed, 358 insertions(+), 30 deletions(-) create mode 100644 numpy/core/src/multiarray/datetime_busdaydef.c create mode 100644 numpy/core/src/multiarray/datetime_busdaydef.h diff --git a/numpy/core/SConscript b/numpy/core/SConscript index 5b25e0b9ca8c..37230e50af3c 100644 --- a/numpy/core/SConscript +++ b/numpy/core/SConscript @@ -440,6 +440,7 @@ if ENABLE_SEPARATE_COMPILATION: pjoin('src', 'multiarray', 'arrayobject.c'), pjoin('src', 'multiarray', 'datetime.c'), pjoin('src', 'multiarray', 'datetime_busday.c'), + pjoin('src', 'multiarray', 'datetime_busdaydef.c'), pjoin('src', 'multiarray', 'numpyos.c'), pjoin('src', 'multiarray', 'flagsobject.c'), pjoin('src', 'multiarray', 'descriptor.c'), diff --git a/numpy/core/code_generators/genapi.py b/numpy/core/code_generators/genapi.py index 4b68ba40cb93..21e6444e6cb7 100644 --- a/numpy/core/code_generators/genapi.py +++ b/numpy/core/code_generators/genapi.py @@ -45,6 +45,7 @@ join('multiarray', 'buffer.c'), join('multiarray', 'datetime.c'), join('multiarray', 'datetime_busday.c'), + join('multiarray', 'datetime_busdaydef.c'), join('multiarray', 'nditer.c.src'), join('multiarray', 'nditer_pywrap.c'), join('multiarray', 'einsum.c.src'), diff --git a/numpy/core/numerictypes.py b/numpy/core/numerictypes.py index f287e1774c1e..21f189b6a4e1 100644 --- a/numpy/core/numerictypes.py +++ b/numpy/core/numerictypes.py @@ -92,11 +92,12 @@ __all__ = ['sctypeDict', 'sctypeNA', 'typeDict', 'typeNA', 'sctypes', 'ScalarType', 'obj2sctype', 'cast', 'nbytes', 'sctype2char', 'maximum_sctype', 'issctype', 'typecodes', 'find_common_type', - 'issubdtype','datetime_data','datetime_as_string', 'busday_offset'] + 'issubdtype', 'datetime_data','datetime_as_string', + 'busday_offset', 'busdaydef'] from numpy.core.multiarray import typeinfo, ndarray, array, \ empty, dtype, datetime_data, datetime_as_string, \ - busday_offset + busday_offset, busdaydef import types as _types import sys diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 8e3bff5b5f0c..0db4698ed187 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -748,6 +748,7 @@ def get_mathlib_info(*args): join('src', 'multiarray', 'buffer.c'), join('src', 'multiarray', 'datetime.c'), join('src', 'multiarray', 'datetime_busday.c'), + join('src', 'multiarray', 'datetime_busdaydef.c'), join('src', 'multiarray', 'numpyos.c'), join('src', 'multiarray', 'conversion_utils.c'), join('src', 'multiarray', 'flagsobject.c'), diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index 02618a9afda9..ce38a035d1d9 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -20,6 +20,7 @@ #include "lowlevel_strided_loops.h" #include "_datetime.h" #include "datetime_busday.h" +#include "datetime_busdaydef.h" /* * Returns 1 if the date is a holiday (contained in the sorted @@ -571,7 +572,7 @@ PyArray_BusDayRollConverter(PyObject *roll_in, NPY_BUSDAY_ROLL *roll) return 1; } -static int +NPY_NO_EXPORT int PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask) { PyObject *obj = weekmask_in; @@ -748,17 +749,6 @@ qsort_datetime_compare(const void *elem1, const void *elem2) return (e1 < e2) ? -1 : (e1 == e2) ? 0 : 1; } -/* - * A list of holidays, which should sorted, not contain any - * duplicates or NaTs, and not include any days already excluded - * by the associated weekmask. - * - * The data is manually managed with PyArray_malloc/PyArray_free. - */ -typedef struct { - npy_datetime *begin, *end; -} npy_holidayslist; - /* * Sorts the the array of dates provided in place and removes * NaT, duplicates and any date which is already excluded on account @@ -767,7 +757,7 @@ typedef struct { * Returns the number of dates left after removing weekmask-excluded * dates. */ -static void +NPY_NO_EXPORT void normalize_holidays_list(npy_holidayslist *holidays, npy_bool *weekmask) { npy_datetime *dates = holidays->begin; @@ -815,7 +805,7 @@ normalize_holidays_list(npy_holidayslist *holidays, npy_bool *weekmask) * know the weekmask. You must call 'normalize_holiday_list' * on the result before using it. */ -static int +NPY_NO_EXPORT int PyArray_HolidaysConverter(PyObject *dates_in, npy_holidayslist *holidays) { PyArrayObject *dates = NULL; @@ -901,25 +891,52 @@ array_busday_offset(PyObject *NPY_UNUSED(self), char *kwlist[] = {"dates", "offsets", "roll", "weekmask", "holidays", "busdaydef", "out", NULL}; - PyObject *dates_in = NULL, *offsets_in = NULL, - *busdaydef_in = NULL, *out_in = NULL; + PyObject *dates_in = NULL, *offsets_in = NULL, *out_in = NULL; PyArrayObject *dates = NULL, *offsets = NULL, *out = NULL, *ret; NPY_BUSDAY_ROLL roll = NPY_BUSDAY_RAISE; - npy_bool weekmask[7] = {1, 1, 1, 1, 1, 0, 0}; + npy_bool weekmask[7] = {2, 1, 1, 1, 1, 0, 0}; + PyArray_BusinessDayDef *busdaydef = NULL; int i, busdays_in_weekmask; npy_holidayslist holidays = {NULL, NULL}; + int allocated_holidays = 1; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O&O&O&OO:busday_offset", kwlist, + "OO|O&O&O&O!O:busday_offset", kwlist, &dates_in, &offsets_in, &PyArray_BusDayRollConverter, &roll, &PyArray_WeekMaskConverter, &weekmask[0], &PyArray_HolidaysConverter, &holidays, - &busdaydef_in, + &NpyBusinessDayDef_Type, &busdaydef, &out_in)) { - return NULL; + goto fail; + } + + /* Make sure only one of the weekmask/holidays and busdaydef is supplied */ + if (busdaydef != NULL) { + if (weekmask[0] != 2 || holidays.begin != NULL) { + PyErr_SetString(PyExc_ValueError, + "Cannot supply both the weekmask/holidays and the " + "busdaydef parameters to busday_offset()"); + goto fail; + } + + /* Indicate that the holidays weren't allocated by us */ + allocated_holidays = 0; + + /* Copy the weekmask/holidays data */ + memcpy(weekmask, busdaydef->weekmask, 7); + holidays = busdaydef->holidays; + } + else { + /* + * Fix up the weekmask from the uninitialized + * signal value to a proper default. + */ + if (weekmask[0] == 2) { + weekmask[0] = 1; + } } /* Make 'dates' into an array */ @@ -928,16 +945,16 @@ array_busday_offset(PyObject *NPY_UNUSED(self), Py_INCREF(dates); } else { - PyArray_Descr *date_dtype; + PyArray_Descr *datetime_dtype; /* Use the datetime dtype with generic units so it fills it in */ - date_dtype = PyArray_DescrFromType(NPY_DATETIME); - if (date_dtype == NULL) { + datetime_dtype = PyArray_DescrFromType(NPY_DATETIME); + if (datetime_dtype == NULL) { goto fail; } - /* This steals the date_dtype reference */ - dates = (PyArrayObject *)PyArray_FromAny(dates_in, date_dtype, + /* This steals the datetime_dtype reference */ + dates = (PyArrayObject *)PyArray_FromAny(dates_in, datetime_dtype, 0, 0, 0, dates_in); if (dates == NULL) { goto fail; @@ -977,7 +994,7 @@ array_busday_offset(PyObject *NPY_UNUSED(self), Py_DECREF(dates); Py_DECREF(offsets); - if (holidays.begin != NULL) { + if (allocated_holidays && holidays.begin != NULL) { PyArray_free(holidays.begin); } @@ -985,7 +1002,7 @@ array_busday_offset(PyObject *NPY_UNUSED(self), fail: Py_XDECREF(dates); Py_XDECREF(offsets); - if (holidays.begin != NULL) { + if (allocated_holidays && holidays.begin != NULL) { PyArray_free(holidays.begin); } diff --git a/numpy/core/src/multiarray/datetime_busday.h b/numpy/core/src/multiarray/datetime_busday.h index aef050463517..8c0dc448210a 100644 --- a/numpy/core/src/multiarray/datetime_busday.h +++ b/numpy/core/src/multiarray/datetime_busday.h @@ -1,6 +1,17 @@ #ifndef _NPY_PRIVATE__DATETIME_BUSDAY_H_ #define _NPY_PRIVATE__DATETIME_BUSDAY_H_ +/* + * A list of holidays, which should sorted, not contain any + * duplicates or NaTs, and not include any days already excluded + * by the associated weekmask. + * + * The data is manually managed with PyArray_malloc/PyArray_free. + */ +typedef struct { + npy_datetime *begin, *end; +} npy_holidayslist; + /* * This is the 'busday_offset' function exposed for calling * from Python. @@ -9,4 +20,34 @@ NPY_NO_EXPORT PyObject * array_busday_offset(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds); +/* + * Converts a Python input into a 7-element weekmask, where 0 means + * weekend and 1 means business day. + */ +NPY_NO_EXPORT int +PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask); + +/* + * Sorts the the array of dates provided in place and removes + * NaT, duplicates and any date which is already excluded on account + * of the weekmask. + * + * Returns the number of dates left after removing weekmask-excluded + * dates. + */ +NPY_NO_EXPORT void +normalize_holidays_list(npy_holidayslist *holidays, npy_bool *weekmask); + +/* + * Converts a Python input into a non-normalized list of holidays. + * + * IMPORTANT: This function can't do the normalization, because it doesn't + * know the weekmask. You must call 'normalize_holiday_list' + * on the result before using it. + */ +NPY_NO_EXPORT int +PyArray_HolidaysConverter(PyObject *dates_in, npy_holidayslist *holidays); + + + #endif diff --git a/numpy/core/src/multiarray/datetime_busdaydef.c b/numpy/core/src/multiarray/datetime_busdaydef.c new file mode 100644 index 000000000000..d27542ea5b55 --- /dev/null +++ b/numpy/core/src/multiarray/datetime_busdaydef.c @@ -0,0 +1,217 @@ +/* + * This file implements an object encapsulating a business day + * definition for accelerating NumPy datetime business day functions. + * + * Written by Mark Wiebe (mwwiebe@gmail.com) + * Copyright (c) 2011 by Enthought, Inc. + * + * See LICENSE.txt for the license. + */ + +#define PY_SSIZE_T_CLEAN +#include + +#define _MULTIARRAYMODULE +#include + +#include "npy_config.h" +#include "numpy/npy_3kcompat.h" + +#include "numpy/arrayscalars.h" +#include "lowlevel_strided_loops.h" +#include "_datetime.h" +#include "datetime_busday.h" +#include "datetime_busdaydef.h" + +static PyObject * +busdaydef_new(PyTypeObject *subtype, + PyObject *NPY_UNUSED(args), PyObject *NPY_UNUSED(kwds)) +{ + PyArray_BusinessDayDef *self; + + self = (PyArray_BusinessDayDef *)subtype->tp_alloc(subtype, 0); + if (self != NULL) { + /* Set the weekmask to the default */ + self->weekmask[0] = 1; + self->weekmask[1] = 1; + self->weekmask[2] = 1; + self->weekmask[3] = 1; + self->weekmask[4] = 1; + self->weekmask[5] = 0; + self->weekmask[6] = 0; + + /* Start with an empty holidays list */ + self->holidays.begin = NULL; + self->holidays.end = NULL; + } + + return (PyObject *)self; +} + +static int +busdaydef_init(PyArray_BusinessDayDef *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"weekmask", "holidays", NULL}; + + /* Reset the weekmask to the default */ + self->weekmask[0] = 1; + self->weekmask[1] = 1; + self->weekmask[2] = 1; + self->weekmask[3] = 1; + self->weekmask[4] = 1; + self->weekmask[5] = 0; + self->weekmask[6] = 0; + + /* Clear the holidays if necessary */ + if (self->holidays.begin != NULL) { + PyArray_free(self->holidays.begin); + self->holidays.begin = NULL; + self->holidays.end = NULL; + } + + /* Parse the parameters */ + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "|O&O&", kwlist, + &PyArray_WeekMaskConverter, &self->weekmask[0], + &PyArray_HolidaysConverter, &self->holidays)) { + return -1; + } + + /* Normalize the holidays list */ + normalize_holidays_list(&self->holidays, self->weekmask); + + return 0; +} + +static void +busdaydef_dealloc(PyArray_BusinessDayDef *self) +{ + /* Clear the holidays */ + if (self->holidays.begin != NULL) { + PyArray_free(self->holidays.begin); + self->holidays.begin = NULL; + self->holidays.end = NULL; + } + + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static PyObject * +busdaydef_weekmask_get(PyArray_BusinessDayDef *self) +{ + PyArrayObject *ret; + npy_intp size = 7; + + /* Allocate a 7-element boolean array */ + ret = (PyArrayObject *)PyArray_SimpleNew(1, &size, NPY_BOOL); + if (ret == NULL) { + return NULL; + } + + /* Copy the weekmask data */ + memcpy(PyArray_DATA(ret), self->weekmask, 7); + + return (PyObject *)ret; +} + +static PyObject * +busdaydef_holidays_get(PyArray_BusinessDayDef *self) +{ + PyArrayObject *ret; + PyArray_Descr *date_dtype; + npy_intp size = self->holidays.end - self->holidays.begin; + + /* Create a date dtype */ + date_dtype = create_datetime_dtype_with_unit(NPY_DATETIME, NPY_FR_D); + if (date_dtype == NULL) { + return NULL; + } + + /* Allocate a date array (this steals the date_dtype reference) */ + ret = (PyArrayObject *)PyArray_SimpleNewFromDescr(1, &size, date_dtype); + if (ret == NULL) { + return NULL; + } + + /* Copy the holidays */ + if (size > 0) { + memcpy(PyArray_DATA(ret), self->holidays.begin, + size * sizeof(npy_datetime)); + } + + return (PyObject *)ret; +} + +static PyGetSetDef busdaydef_getsets[] = { + {"weekmask", + (getter)busdaydef_weekmask_get, + NULL, NULL, NULL}, + {"holidays", + (getter)busdaydef_holidays_get, + NULL, NULL, NULL}, + + {NULL, NULL, NULL, NULL, NULL} +}; + +NPY_NO_EXPORT PyTypeObject NpyBusinessDayDef_Type = { +#if defined(NPY_PY3K) + PyVarObject_HEAD_INIT(NULL, 0) +#else + PyObject_HEAD_INIT(NULL) + 0, /* ob_size */ +#endif + "numpy.busdaydef", /* tp_name */ + sizeof(PyArray_BusinessDayDef), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)busdaydef_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ +#if defined(NPY_PY3K) + 0, /* tp_reserved */ +#else + 0, /* tp_compare */ +#endif + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + busdaydef_getsets, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)busdaydef_init, /* tp_init */ + 0, /* tp_alloc */ + busdaydef_new, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ +#if PY_VERSION_HEX >= 0x02060000 + 0, /* tp_version_tag */ +#endif +}; + diff --git a/numpy/core/src/multiarray/datetime_busdaydef.h b/numpy/core/src/multiarray/datetime_busdaydef.h new file mode 100644 index 000000000000..262143e70fa4 --- /dev/null +++ b/numpy/core/src/multiarray/datetime_busdaydef.h @@ -0,0 +1,15 @@ +#ifndef _NPY_PRIVATE__DATETIME_BUSDAYDEF_H_ +#define _NPY_PRIVATE__DATETIME_BUSDAYDEF_H_ + +typedef struct { + PyObject_HEAD + npy_holidayslist holidays; + npy_bool weekmask[7]; +} PyArray_BusinessDayDef; + +NPY_NO_EXPORT PyTypeObject NpyBusinessDayDef_Type; + +#define NpyBusinessDayDef_Check(op) PyObject_TypeCheck(op, \ + &NpyBusinessDayDef_Type) + +#endif diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index fb3b0c352764..5b973bc8d154 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -46,6 +46,7 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0; #include "nditer_pywrap.h" #include "_datetime.h" #include "datetime_busday.h" +#include "datetime_busdaydef.h" /* Only here for API compatibility */ NPY_NO_EXPORT PyTypeObject PyBigArray_Type; @@ -3915,6 +3916,10 @@ PyMODINIT_FUNC initmultiarray(void) { if (PyType_Ready(&PyArrayFlags_Type) < 0) { return RETVAL; } + NpyBusinessDayDef_Type.tp_new = PyType_GenericNew; + if (PyType_Ready(&NpyBusinessDayDef_Type) < 0) { + return RETVAL; + } /* FIXME * There is no error handling here */ @@ -3994,6 +3999,10 @@ PyMODINIT_FUNC initmultiarray(void) { Py_INCREF(&PyArrayFlags_Type); PyDict_SetItemString(d, "flagsobj", (PyObject *)&PyArrayFlags_Type); + /* Business day definition object */ + Py_INCREF(&NpyBusinessDayDef_Type); + PyDict_SetItemString(d, "busdaydef", (PyObject *)&NpyBusinessDayDef_Type); + set_flaginfo(d); if (set_typeinfo(d) != 0) { diff --git a/numpy/core/src/multiarray/multiarraymodule_onefile.c b/numpy/core/src/multiarray/multiarraymodule_onefile.c index 17c4c739d85e..5f7cf17f3e73 100644 --- a/numpy/core/src/multiarray/multiarraymodule_onefile.c +++ b/numpy/core/src/multiarray/multiarraymodule_onefile.c @@ -12,6 +12,7 @@ #include "datetime.c" #include "datetime_busday.c" +#include "datetime_busdaydef.c" #include "arraytypes.c" #include "hashdescr.c" diff --git a/numpy/core/src/multiarray/nditer_pywrap.c b/numpy/core/src/multiarray/nditer_pywrap.c index 5227f9c1e9fe..b89da50fca90 100644 --- a/numpy/core/src/multiarray/nditer_pywrap.c +++ b/numpy/core/src/multiarray/nditer_pywrap.c @@ -2401,7 +2401,7 @@ static PyGetSetDef npyiter_getsets[] = { (getter)npyiter_finished_get, NULL, NULL, NULL}, - {NULL, NULL, NULL, NULL, NULL}, + {NULL, NULL, NULL, NULL, NULL} }; NPY_NO_EXPORT PySequenceMethods npyiter_as_sequence = { diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index 9d80200b59b0..a45994691320 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -1206,6 +1206,17 @@ def test_datetime_busday_offset(self): assert_equal(np.busday_offset('2007-04-07', -11, weekmask='SatSun'), np.datetime64('2007-02-25')) + def test_datetime_busdaydef(self): + # Check that it removes NaT, duplicates, and weekends + # and sorts the result. + bdd = np.busdaydef( + holidays=['NaT', '2011-01-17', '2011-03-06', 'NaT', + '2011-12-26','2011-05-30','2011-01-17']) + assert_equal(bdd.holidays, + np.array(['2011-01-17','2011-05-30','2011-12-26'], dtype='M8')) + # Default M-F weekmask + assert_equal(bdd.weekmask, np.array([1,1,1,1,1,0,0], dtype='?')) + def test_datetime_busday_holidays_offset(self): # With exactly one holiday assert_equal( @@ -1276,6 +1287,7 @@ def test_datetime_busday_holidays_offset(self): holidays=['2011-10-10', '2011-11-11', '2011-11-24', '2011-12-25', '2011-05-30', '2011-02-21', '2011-12-26', '2012-01-02'] + bdd = np.busdaydef(weekmask='1111100', holidays=holidays) assert_equal( np.busday_offset('2011-10-03', 4, holidays=holidays), np.busday_offset('2011-10-03', 4)) @@ -1306,6 +1318,9 @@ def test_datetime_busday_holidays_offset(self): assert_equal( np.busday_offset('2011-10-03', 61, holidays=holidays), np.busday_offset('2011-10-03', 61 + 5)) + assert_equal( + np.busday_offset('2011-10-03', 61, busdaydef=bdd), + np.busday_offset('2011-10-03', 61 + 5)) # A bigger backward jump across more than one week/holiday assert_equal( np.busday_offset('2012-01-03', -1, holidays=holidays), @@ -1334,6 +1349,15 @@ def test_datetime_busday_holidays_offset(self): assert_equal( np.busday_offset('2012-01-03', -57, holidays=holidays), np.busday_offset('2012-01-03', -57 - 5)) + assert_equal( + np.busday_offset('2012-01-03', -57, busdaydef=bdd), + np.busday_offset('2012-01-03', -57 - 5)) + + # Can't supply both a weekmask/holidays and busdaydef + assert_raises(ValueError, np.busday_offset, '2012-01-03', -15, + weekmask='1111100', busdaydef=bdd) + assert_raises(ValueError, np.busday_offset, '2012-01-03', -15, + holidays=holidays, busdaydef=bdd) # Roll with the holidays assert_equal( From e164d9425783a102e7bfdedcd04fcafcbcdc993c Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Tue, 14 Jun 2011 09:47:35 -0500 Subject: [PATCH 21/34] ENH: datetime-bday: Cache the business day count in the weekmask in busdaydef --- numpy/core/src/multiarray/datetime_busday.c | 32 ++++++++----- .../core/src/multiarray/datetime_busdaydef.c | 45 +++++++++++++------ .../core/src/multiarray/datetime_busdaydef.h | 1 + numpy/core/tests/test_datetime.py | 3 ++ 4 files changed, 56 insertions(+), 25 deletions(-) diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index ce38a035d1d9..21647d1ea11f 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -475,6 +475,15 @@ business_day_offset(PyArrayObject *dates, PyArrayObject *offsets, return ret; } +NPY_NO_EXPORT npy_intp +business_day_count(PyArrayObject *dates_begin, PyArrayObject *dates_end, + PyArrayObject *out, + NPY_BUSDAY_ROLL roll, + npy_bool *weekmask, int busdays_in_weekmask, + npy_datetime *holidays_begin, npy_datetime *holidays_end) +{ +} + static int PyArray_BusDayRollConverter(PyObject *roll_in, NPY_BUSDAY_ROLL *roll) { @@ -925,9 +934,10 @@ array_busday_offset(PyObject *NPY_UNUSED(self), /* Indicate that the holidays weren't allocated by us */ allocated_holidays = 0; - /* Copy the weekmask/holidays data */ - memcpy(weekmask, busdaydef->weekmask, 7); + /* Copy the private normalized weekmask/holidays data */ holidays = busdaydef->holidays; + busdays_in_weekmask = busdaydef->busdays_in_weekmask; + memcpy(weekmask, busdaydef->weekmask, 7); } else { /* @@ -937,6 +947,15 @@ array_busday_offset(PyObject *NPY_UNUSED(self), if (weekmask[0] == 2) { weekmask[0] = 1; } + + /* Count the number of business days in a week */ + busdays_in_weekmask = 0; + for (i = 0; i < 7; ++i) { + busdays_in_weekmask += weekmask[i]; + } + + /* The holidays list must be normalized before using it */ + normalize_holidays_list(&holidays, weekmask); } /* Make 'dates' into an array */ @@ -979,15 +998,6 @@ array_busday_offset(PyObject *NPY_UNUSED(self), out = (PyArrayObject *)out_in; } - /* Count the number of business days in a week */ - busdays_in_weekmask = 0; - for (i = 0; i < 7; ++i) { - busdays_in_weekmask += weekmask[i]; - } - - /* The holidays list must be normalized before using it */ - normalize_holidays_list(&holidays, weekmask); - ret = business_day_offset(dates, offsets, out, roll, weekmask, busdays_in_weekmask, holidays.begin, holidays.end); diff --git a/numpy/core/src/multiarray/datetime_busdaydef.c b/numpy/core/src/multiarray/datetime_busdaydef.c index d27542ea5b55..74b3eeef7a30 100644 --- a/numpy/core/src/multiarray/datetime_busdaydef.c +++ b/numpy/core/src/multiarray/datetime_busdaydef.c @@ -31,7 +31,12 @@ busdaydef_new(PyTypeObject *subtype, self = (PyArray_BusinessDayDef *)subtype->tp_alloc(subtype, 0); if (self != NULL) { - /* Set the weekmask to the default */ + /* Start with an empty holidays list */ + self->holidays.begin = NULL; + self->holidays.end = NULL; + + /* Set the weekmask to the default */ + self->busdays_in_weekmask = 5; self->weekmask[0] = 1; self->weekmask[1] = 1; self->weekmask[2] = 1; @@ -39,10 +44,6 @@ busdaydef_new(PyTypeObject *subtype, self->weekmask[4] = 1; self->weekmask[5] = 0; self->weekmask[6] = 0; - - /* Start with an empty holidays list */ - self->holidays.begin = NULL; - self->holidays.end = NULL; } return (PyObject *)self; @@ -52,8 +53,17 @@ static int busdaydef_init(PyArray_BusinessDayDef *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"weekmask", "holidays", NULL}; + int i, busdays_in_weekmask; + + /* Clear the holidays if necessary */ + if (self->holidays.begin != NULL) { + PyArray_free(self->holidays.begin); + self->holidays.begin = NULL; + self->holidays.end = NULL; + } /* Reset the weekmask to the default */ + self->busdays_in_weekmask = 5; self->weekmask[0] = 1; self->weekmask[1] = 1; self->weekmask[2] = 1; @@ -62,24 +72,31 @@ busdaydef_init(PyArray_BusinessDayDef *self, PyObject *args, PyObject *kwds) self->weekmask[5] = 0; self->weekmask[6] = 0; - /* Clear the holidays if necessary */ - if (self->holidays.begin != NULL) { - PyArray_free(self->holidays.begin); - self->holidays.begin = NULL; - self->holidays.end = NULL; - } - /* Parse the parameters */ if (!PyArg_ParseTupleAndKeywords(args, kwds, - "|O&O&", kwlist, + "|O&O&:busdaydef", kwlist, &PyArray_WeekMaskConverter, &self->weekmask[0], &PyArray_HolidaysConverter, &self->holidays)) { return -1; } - + + /* Count the number of business days in a week */ + busdays_in_weekmask = 0; + for (i = 0; i < 7; ++i) { + busdays_in_weekmask += self->weekmask[i]; + } + self->busdays_in_weekmask = busdays_in_weekmask; + /* Normalize the holidays list */ normalize_holidays_list(&self->holidays, self->weekmask); + if (self->busdays_in_weekmask == 0) { + PyErr_SetString(PyExc_ValueError, + "Cannot construct a numpy.busdaydef with a weekmask of " + "all zeros"); + return -1; + } + return 0; } diff --git a/numpy/core/src/multiarray/datetime_busdaydef.h b/numpy/core/src/multiarray/datetime_busdaydef.h index 262143e70fa4..dafc5c07d7e7 100644 --- a/numpy/core/src/multiarray/datetime_busdaydef.h +++ b/numpy/core/src/multiarray/datetime_busdaydef.h @@ -4,6 +4,7 @@ typedef struct { PyObject_HEAD npy_holidayslist holidays; + int busdays_in_weekmask; npy_bool weekmask[7]; } PyArray_BusinessDayDef; diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index a45994691320..ecac60e5dd37 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -1217,6 +1217,9 @@ def test_datetime_busdaydef(self): # Default M-F weekmask assert_equal(bdd.weekmask, np.array([1,1,1,1,1,0,0], dtype='?')) + # All-zeros weekmask should raise + assert_raises(ValueError, np.busdaydef, weekmask=[0,0,0,0,0,0,0]) + def test_datetime_busday_holidays_offset(self): # With exactly one holiday assert_equal( From f7ae666d8747e850e49f7d7b5a0daa1c5519d8be Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Tue, 14 Jun 2011 09:52:24 -0500 Subject: [PATCH 22/34] ENH: datetime-bday: Move the weekmask and holidays list convertors to busdaydef --- numpy/core/src/multiarray/datetime_busday.c | 310 +---------------- numpy/core/src/multiarray/datetime_busday.h | 29 -- .../core/src/multiarray/datetime_busdaydef.c | 322 +++++++++++++++++- .../core/src/multiarray/datetime_busdaydef.h | 32 +- 4 files changed, 347 insertions(+), 346 deletions(-) diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index 21647d1ea11f..08b3072237b2 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -581,314 +581,6 @@ PyArray_BusDayRollConverter(PyObject *roll_in, NPY_BUSDAY_ROLL *roll) return 1; } -NPY_NO_EXPORT int -PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask) -{ - PyObject *obj = weekmask_in; - - /* Make obj into an ASCII string if it is UNICODE */ - Py_INCREF(obj); - if (PyUnicode_Check(obj)) { - /* accept unicode input */ - PyObject *obj_str; - obj_str = PyUnicode_AsASCIIString(obj); - if (obj_str == NULL) { - Py_DECREF(obj); - return 0; - } - Py_DECREF(obj); - obj = obj_str; - } - - if (PyBytes_Check(obj)) { - char *str; - Py_ssize_t len; - - if (PyBytes_AsStringAndSize(obj, &str, &len) < 0) { - Py_DECREF(obj); - return 0; - } - - /* Length 7 is a string like "1111100" */ - if (len == 7) { - int i; - for (i = 0; i < 7; ++i) { - switch(str[i]) { - case '0': - weekmask[i] = 0; - break; - case '1': - weekmask[i] = 1; - break; - default: - goto invalid_weekmask_string; - } - } - - goto finish; - } - /* Length divisible by 3 is a string like "Mon" or "MonWedFri" */ - else if (len % 3 == 0) { - int i; - memset(weekmask, 0, 7); - for (i = 0; i < len; i += 3) { - switch (str[i]) { - case 'M': - if (str[i+1] == 'o' && str[i+2] == 'n') { - weekmask[0] = 1; - } - else { - goto invalid_weekmask_string; - } - break; - case 'T': - if (str[i+1] == 'u' && str[i+2] == 'e') { - weekmask[1] = 1; - } - else if (str[i+1] == 'h' && str[i+2] == 'u') { - weekmask[3] = 1; - } - else { - goto invalid_weekmask_string; - } - break; - case 'W': - if (str[i+1] == 'e' && str[i+2] == 'd') { - weekmask[2] = 1; - } - else { - goto invalid_weekmask_string; - } - break; - case 'F': - if (str[i+1] == 'r' && str[i+2] == 'i') { - weekmask[4] = 1; - } - else { - goto invalid_weekmask_string; - } - break; - case 'S': - if (str[i+1] == 'a' && str[i+2] == 't') { - weekmask[5] = 1; - } - else if (str[i+1] == 'u' && str[i+2] == 'n') { - weekmask[6] = 1; - } - else { - goto invalid_weekmask_string; - } - break; - } - } - - goto finish; - } - -invalid_weekmask_string: - PyErr_Format(PyExc_ValueError, - "Invalid business day weekmask string \"%s\"", - str); - Py_DECREF(obj); - return 0; - } - /* Something like [1,1,1,1,1,0,0] */ - else if (PySequence_Check(obj)) { - if (PySequence_Size(obj) != 7 || - (PyArray_Check(obj) && PyArray_NDIM(obj) != 1)) { - PyErr_SetString(PyExc_ValueError, - "A business day weekmask array must have length 7"); - Py_DECREF(obj); - return 0; - } - else { - int i; - PyObject *f; - - for (i = 0; i < 7; ++i) { - long val; - - f = PySequence_GetItem(obj, i); - if (f == NULL) { - Py_DECREF(obj); - return 0; - } - - val = PyInt_AsLong(f); - if (val == -1 && PyErr_Occurred()) { - Py_DECREF(obj); - return 0; - } - if (val == 0) { - weekmask[i] = 0; - } - else if (val == 1) { - weekmask[i] = 1; - } - else { - PyErr_SetString(PyExc_ValueError, - "A business day weekmask array must have all " - "1's and 0's"); - Py_DECREF(obj); - return 0; - } - } - - goto finish; - } - } - - PyErr_SetString(PyExc_ValueError, - "Couldn't convert object into a business day weekmask"); - Py_DECREF(obj); - return 0; - - -finish: - Py_DECREF(obj); - return 1; -} - -static int -qsort_datetime_compare(const void *elem1, const void *elem2) -{ - npy_datetime e1 = *(npy_datetime *)elem1; - npy_datetime e2 = *(npy_datetime *)elem2; - - return (e1 < e2) ? -1 : (e1 == e2) ? 0 : 1; -} - -/* - * Sorts the the array of dates provided in place and removes - * NaT, duplicates and any date which is already excluded on account - * of the weekmask. - * - * Returns the number of dates left after removing weekmask-excluded - * dates. - */ -NPY_NO_EXPORT void -normalize_holidays_list(npy_holidayslist *holidays, npy_bool *weekmask) -{ - npy_datetime *dates = holidays->begin; - npy_intp count = holidays->end - dates; - - npy_datetime lastdate = NPY_DATETIME_NAT; - npy_intp trimcount, i; - int day_of_week; - - /* Sort the dates */ - qsort(dates, count, sizeof(npy_datetime), &qsort_datetime_compare); - - /* Sweep throught the array, eliminating unnecessary values */ - trimcount = 0; - for (i = 0; i < count; ++i) { - npy_datetime date = dates[i]; - - /* Skip any NaT or duplicate */ - if (date != NPY_DATETIME_NAT && date != lastdate) { - /* Get the day of the week (1970-01-05 is Monday) */ - day_of_week = (int)((date - 4) % 7); - if (day_of_week < 0) { - day_of_week += 7; - } - - /* - * If the holiday falls on a possible business day, - * then keep it. - */ - if (weekmask[day_of_week] == 1) { - dates[trimcount++] = date; - lastdate = date; - } - } - } - - /* Adjust the end of the holidays array */ - holidays->end = dates + trimcount; -} - -/* - * Converts a Python input into a non-normalized list of holidays. - * - * IMPORTANT: This function can't do the normalization, because it doesn't - * know the weekmask. You must call 'normalize_holiday_list' - * on the result before using it. - */ -NPY_NO_EXPORT int -PyArray_HolidaysConverter(PyObject *dates_in, npy_holidayslist *holidays) -{ - PyArrayObject *dates = NULL; - PyArray_Descr *date_dtype = NULL; - npy_intp count; - - /* Make 'dates' into an array */ - if (PyArray_Check(dates_in)) { - dates = (PyArrayObject *)dates_in; - Py_INCREF(dates); - } - else { - PyArray_Descr *datetime_dtype; - - /* Use the datetime dtype with generic units so it fills it in */ - datetime_dtype = PyArray_DescrFromType(NPY_DATETIME); - if (datetime_dtype == NULL) { - goto fail; - } - - /* This steals the datetime_dtype reference */ - dates = (PyArrayObject *)PyArray_FromAny(dates_in, datetime_dtype, - 0, 0, 0, dates_in); - if (dates == NULL) { - goto fail; - } - } - - date_dtype = create_datetime_dtype_with_unit(NPY_DATETIME, NPY_FR_D); - if (date_dtype == NULL) { - goto fail; - } - - if (!PyArray_CanCastTypeTo(PyArray_DESCR(dates), - date_dtype, NPY_SAFE_CASTING)) { - PyErr_SetString(PyExc_ValueError, "Cannot safely convert " - "provided holidays input into an array of dates"); - goto fail; - } - if (PyArray_NDIM(dates) != 1) { - PyErr_SetString(PyExc_ValueError, "holidays must be a provided " - "as a one-dimensional array"); - goto fail; - } - - /* Allocate the memory for the dates */ - count = PyArray_DIM(dates, 0); - holidays->begin = PyArray_malloc(sizeof(npy_datetime) * count); - if (holidays->begin == NULL) { - PyErr_NoMemory(); - goto fail; - } - holidays->end = holidays->begin + count; - - /* Cast the data into a raw date array */ - if (PyArray_CastRawArrays(count, - PyArray_BYTES(dates), (char *)holidays->begin, - PyArray_STRIDE(dates, 0), sizeof(npy_datetime), - PyArray_DESCR(dates), date_dtype, - 0) != NPY_SUCCEED) { - goto fail; - } - - Py_DECREF(dates); - Py_DECREF(date_dtype); - - return 1; - -fail: - Py_XDECREF(dates); - Py_XDECREF(date_dtype); - return 0; -} - /* * This is the 'busday_offset' function exposed for calling * from Python. @@ -905,7 +597,7 @@ array_busday_offset(PyObject *NPY_UNUSED(self), PyArrayObject *dates = NULL, *offsets = NULL, *out = NULL, *ret; NPY_BUSDAY_ROLL roll = NPY_BUSDAY_RAISE; npy_bool weekmask[7] = {2, 1, 1, 1, 1, 0, 0}; - PyArray_BusinessDayDef *busdaydef = NULL; + NpyBusinessDayDef *busdaydef = NULL; int i, busdays_in_weekmask; npy_holidayslist holidays = {NULL, NULL}; int allocated_holidays = 1; diff --git a/numpy/core/src/multiarray/datetime_busday.h b/numpy/core/src/multiarray/datetime_busday.h index 8c0dc448210a..f1cb68577f34 100644 --- a/numpy/core/src/multiarray/datetime_busday.h +++ b/numpy/core/src/multiarray/datetime_busday.h @@ -20,34 +20,5 @@ NPY_NO_EXPORT PyObject * array_busday_offset(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds); -/* - * Converts a Python input into a 7-element weekmask, where 0 means - * weekend and 1 means business day. - */ -NPY_NO_EXPORT int -PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask); - -/* - * Sorts the the array of dates provided in place and removes - * NaT, duplicates and any date which is already excluded on account - * of the weekmask. - * - * Returns the number of dates left after removing weekmask-excluded - * dates. - */ -NPY_NO_EXPORT void -normalize_holidays_list(npy_holidayslist *holidays, npy_bool *weekmask); - -/* - * Converts a Python input into a non-normalized list of holidays. - * - * IMPORTANT: This function can't do the normalization, because it doesn't - * know the weekmask. You must call 'normalize_holiday_list' - * on the result before using it. - */ -NPY_NO_EXPORT int -PyArray_HolidaysConverter(PyObject *dates_in, npy_holidayslist *holidays); - - #endif diff --git a/numpy/core/src/multiarray/datetime_busdaydef.c b/numpy/core/src/multiarray/datetime_busdaydef.c index 74b3eeef7a30..396bf82bec9e 100644 --- a/numpy/core/src/multiarray/datetime_busdaydef.c +++ b/numpy/core/src/multiarray/datetime_busdaydef.c @@ -23,13 +23,321 @@ #include "datetime_busday.h" #include "datetime_busdaydef.h" +NPY_NO_EXPORT int +PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask) +{ + PyObject *obj = weekmask_in; + + /* Make obj into an ASCII string if it is UNICODE */ + Py_INCREF(obj); + if (PyUnicode_Check(obj)) { + /* accept unicode input */ + PyObject *obj_str; + obj_str = PyUnicode_AsASCIIString(obj); + if (obj_str == NULL) { + Py_DECREF(obj); + return 0; + } + Py_DECREF(obj); + obj = obj_str; + } + + if (PyBytes_Check(obj)) { + char *str; + Py_ssize_t len; + + if (PyBytes_AsStringAndSize(obj, &str, &len) < 0) { + Py_DECREF(obj); + return 0; + } + + /* Length 7 is a string like "1111100" */ + if (len == 7) { + int i; + for (i = 0; i < 7; ++i) { + switch(str[i]) { + case '0': + weekmask[i] = 0; + break; + case '1': + weekmask[i] = 1; + break; + default: + goto invalid_weekmask_string; + } + } + + goto finish; + } + /* Length divisible by 3 is a string like "Mon" or "MonWedFri" */ + else if (len % 3 == 0) { + int i; + memset(weekmask, 0, 7); + for (i = 0; i < len; i += 3) { + switch (str[i]) { + case 'M': + if (str[i+1] == 'o' && str[i+2] == 'n') { + weekmask[0] = 1; + } + else { + goto invalid_weekmask_string; + } + break; + case 'T': + if (str[i+1] == 'u' && str[i+2] == 'e') { + weekmask[1] = 1; + } + else if (str[i+1] == 'h' && str[i+2] == 'u') { + weekmask[3] = 1; + } + else { + goto invalid_weekmask_string; + } + break; + case 'W': + if (str[i+1] == 'e' && str[i+2] == 'd') { + weekmask[2] = 1; + } + else { + goto invalid_weekmask_string; + } + break; + case 'F': + if (str[i+1] == 'r' && str[i+2] == 'i') { + weekmask[4] = 1; + } + else { + goto invalid_weekmask_string; + } + break; + case 'S': + if (str[i+1] == 'a' && str[i+2] == 't') { + weekmask[5] = 1; + } + else if (str[i+1] == 'u' && str[i+2] == 'n') { + weekmask[6] = 1; + } + else { + goto invalid_weekmask_string; + } + break; + } + } + + goto finish; + } + +invalid_weekmask_string: + PyErr_Format(PyExc_ValueError, + "Invalid business day weekmask string \"%s\"", + str); + Py_DECREF(obj); + return 0; + } + /* Something like [1,1,1,1,1,0,0] */ + else if (PySequence_Check(obj)) { + if (PySequence_Size(obj) != 7 || + (PyArray_Check(obj) && PyArray_NDIM(obj) != 1)) { + PyErr_SetString(PyExc_ValueError, + "A business day weekmask array must have length 7"); + Py_DECREF(obj); + return 0; + } + else { + int i; + PyObject *f; + + for (i = 0; i < 7; ++i) { + long val; + + f = PySequence_GetItem(obj, i); + if (f == NULL) { + Py_DECREF(obj); + return 0; + } + + val = PyInt_AsLong(f); + if (val == -1 && PyErr_Occurred()) { + Py_DECREF(obj); + return 0; + } + if (val == 0) { + weekmask[i] = 0; + } + else if (val == 1) { + weekmask[i] = 1; + } + else { + PyErr_SetString(PyExc_ValueError, + "A business day weekmask array must have all " + "1's and 0's"); + Py_DECREF(obj); + return 0; + } + } + + goto finish; + } + } + + PyErr_SetString(PyExc_ValueError, + "Couldn't convert object into a business day weekmask"); + Py_DECREF(obj); + return 0; + + +finish: + Py_DECREF(obj); + return 1; +} + +static int +qsort_datetime_compare(const void *elem1, const void *elem2) +{ + npy_datetime e1 = *(npy_datetime *)elem1; + npy_datetime e2 = *(npy_datetime *)elem2; + + return (e1 < e2) ? -1 : (e1 == e2) ? 0 : 1; +} + +/* + * Sorts the the array of dates provided in place and removes + * NaT, duplicates and any date which is already excluded on account + * of the weekmask. + * + * Returns the number of dates left after removing weekmask-excluded + * dates. + */ +NPY_NO_EXPORT void +normalize_holidays_list(npy_holidayslist *holidays, npy_bool *weekmask) +{ + npy_datetime *dates = holidays->begin; + npy_intp count = holidays->end - dates; + + npy_datetime lastdate = NPY_DATETIME_NAT; + npy_intp trimcount, i; + int day_of_week; + + /* Sort the dates */ + qsort(dates, count, sizeof(npy_datetime), &qsort_datetime_compare); + + /* Sweep throught the array, eliminating unnecessary values */ + trimcount = 0; + for (i = 0; i < count; ++i) { + npy_datetime date = dates[i]; + + /* Skip any NaT or duplicate */ + if (date != NPY_DATETIME_NAT && date != lastdate) { + /* Get the day of the week (1970-01-05 is Monday) */ + day_of_week = (int)((date - 4) % 7); + if (day_of_week < 0) { + day_of_week += 7; + } + + /* + * If the holiday falls on a possible business day, + * then keep it. + */ + if (weekmask[day_of_week] == 1) { + dates[trimcount++] = date; + lastdate = date; + } + } + } + + /* Adjust the end of the holidays array */ + holidays->end = dates + trimcount; +} + +/* + * Converts a Python input into a non-normalized list of holidays. + * + * IMPORTANT: This function can't do the normalization, because it doesn't + * know the weekmask. You must call 'normalize_holiday_list' + * on the result before using it. + */ +NPY_NO_EXPORT int +PyArray_HolidaysConverter(PyObject *dates_in, npy_holidayslist *holidays) +{ + PyArrayObject *dates = NULL; + PyArray_Descr *date_dtype = NULL; + npy_intp count; + + /* Make 'dates' into an array */ + if (PyArray_Check(dates_in)) { + dates = (PyArrayObject *)dates_in; + Py_INCREF(dates); + } + else { + PyArray_Descr *datetime_dtype; + + /* Use the datetime dtype with generic units so it fills it in */ + datetime_dtype = PyArray_DescrFromType(NPY_DATETIME); + if (datetime_dtype == NULL) { + goto fail; + } + + /* This steals the datetime_dtype reference */ + dates = (PyArrayObject *)PyArray_FromAny(dates_in, datetime_dtype, + 0, 0, 0, dates_in); + if (dates == NULL) { + goto fail; + } + } + + date_dtype = create_datetime_dtype_with_unit(NPY_DATETIME, NPY_FR_D); + if (date_dtype == NULL) { + goto fail; + } + + if (!PyArray_CanCastTypeTo(PyArray_DESCR(dates), + date_dtype, NPY_SAFE_CASTING)) { + PyErr_SetString(PyExc_ValueError, "Cannot safely convert " + "provided holidays input into an array of dates"); + goto fail; + } + if (PyArray_NDIM(dates) != 1) { + PyErr_SetString(PyExc_ValueError, "holidays must be a provided " + "as a one-dimensional array"); + goto fail; + } + + /* Allocate the memory for the dates */ + count = PyArray_DIM(dates, 0); + holidays->begin = PyArray_malloc(sizeof(npy_datetime) * count); + if (holidays->begin == NULL) { + PyErr_NoMemory(); + goto fail; + } + holidays->end = holidays->begin + count; + + /* Cast the data into a raw date array */ + if (PyArray_CastRawArrays(count, + PyArray_BYTES(dates), (char *)holidays->begin, + PyArray_STRIDE(dates, 0), sizeof(npy_datetime), + PyArray_DESCR(dates), date_dtype, + 0) != NPY_SUCCEED) { + goto fail; + } + + Py_DECREF(dates); + Py_DECREF(date_dtype); + + return 1; + +fail: + Py_XDECREF(dates); + Py_XDECREF(date_dtype); + return 0; +} + static PyObject * busdaydef_new(PyTypeObject *subtype, PyObject *NPY_UNUSED(args), PyObject *NPY_UNUSED(kwds)) { - PyArray_BusinessDayDef *self; + NpyBusinessDayDef *self; - self = (PyArray_BusinessDayDef *)subtype->tp_alloc(subtype, 0); + self = (NpyBusinessDayDef *)subtype->tp_alloc(subtype, 0); if (self != NULL) { /* Start with an empty holidays list */ self->holidays.begin = NULL; @@ -50,7 +358,7 @@ busdaydef_new(PyTypeObject *subtype, } static int -busdaydef_init(PyArray_BusinessDayDef *self, PyObject *args, PyObject *kwds) +busdaydef_init(NpyBusinessDayDef *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"weekmask", "holidays", NULL}; int i, busdays_in_weekmask; @@ -101,7 +409,7 @@ busdaydef_init(PyArray_BusinessDayDef *self, PyObject *args, PyObject *kwds) } static void -busdaydef_dealloc(PyArray_BusinessDayDef *self) +busdaydef_dealloc(NpyBusinessDayDef *self) { /* Clear the holidays */ if (self->holidays.begin != NULL) { @@ -114,7 +422,7 @@ busdaydef_dealloc(PyArray_BusinessDayDef *self) } static PyObject * -busdaydef_weekmask_get(PyArray_BusinessDayDef *self) +busdaydef_weekmask_get(NpyBusinessDayDef *self) { PyArrayObject *ret; npy_intp size = 7; @@ -132,7 +440,7 @@ busdaydef_weekmask_get(PyArray_BusinessDayDef *self) } static PyObject * -busdaydef_holidays_get(PyArray_BusinessDayDef *self) +busdaydef_holidays_get(NpyBusinessDayDef *self) { PyArrayObject *ret; PyArray_Descr *date_dtype; @@ -178,7 +486,7 @@ NPY_NO_EXPORT PyTypeObject NpyBusinessDayDef_Type = { 0, /* ob_size */ #endif "numpy.busdaydef", /* tp_name */ - sizeof(PyArray_BusinessDayDef), /* tp_basicsize */ + sizeof(NpyBusinessDayDef), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ (destructor)busdaydef_dealloc, /* tp_dealloc */ diff --git a/numpy/core/src/multiarray/datetime_busdaydef.h b/numpy/core/src/multiarray/datetime_busdaydef.h index dafc5c07d7e7..64e5244a8682 100644 --- a/numpy/core/src/multiarray/datetime_busdaydef.h +++ b/numpy/core/src/multiarray/datetime_busdaydef.h @@ -6,11 +6,41 @@ typedef struct { npy_holidayslist holidays; int busdays_in_weekmask; npy_bool weekmask[7]; -} PyArray_BusinessDayDef; +} NpyBusinessDayDef; NPY_NO_EXPORT PyTypeObject NpyBusinessDayDef_Type; #define NpyBusinessDayDef_Check(op) PyObject_TypeCheck(op, \ &NpyBusinessDayDef_Type) +/* + * Converts a Python input into a 7-element weekmask, where 0 means + * weekend and 1 means business day. + */ +NPY_NO_EXPORT int +PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask); + +/* + * Sorts the the array of dates provided in place and removes + * NaT, duplicates and any date which is already excluded on account + * of the weekmask. + * + * Returns the number of dates left after removing weekmask-excluded + * dates. + */ +NPY_NO_EXPORT void +normalize_holidays_list(npy_holidayslist *holidays, npy_bool *weekmask); + +/* + * Converts a Python input into a non-normalized list of holidays. + * + * IMPORTANT: This function can't do the normalization, because it doesn't + * know the weekmask. You must call 'normalize_holiday_list' + * on the result before using it. + */ +NPY_NO_EXPORT int +PyArray_HolidaysConverter(PyObject *dates_in, npy_holidayslist *holidays); + + + #endif From 0418574b0fe6fcc7a70bbfc04a7feef25b3416a6 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Tue, 14 Jun 2011 10:55:54 -0500 Subject: [PATCH 23/34] ENH: datetime-bday: Got busday_count function working --- numpy/core/numerictypes.py | 4 +- numpy/core/src/multiarray/datetime_busday.c | 358 +++++++++++++++++- numpy/core/src/multiarray/datetime_busday.h | 17 +- .../core/src/multiarray/datetime_busdaydef.h | 18 + numpy/core/src/multiarray/multiarraymodule.c | 3 + numpy/core/tests/test_datetime.py | 25 ++ 6 files changed, 406 insertions(+), 19 deletions(-) diff --git a/numpy/core/numerictypes.py b/numpy/core/numerictypes.py index 21f189b6a4e1..0b61b7161663 100644 --- a/numpy/core/numerictypes.py +++ b/numpy/core/numerictypes.py @@ -93,11 +93,11 @@ 'ScalarType', 'obj2sctype', 'cast', 'nbytes', 'sctype2char', 'maximum_sctype', 'issctype', 'typecodes', 'find_common_type', 'issubdtype', 'datetime_data','datetime_as_string', - 'busday_offset', 'busdaydef'] + 'busday_offset', 'busday_count', 'busdaydef'] from numpy.core.multiarray import typeinfo, ndarray, array, \ empty, dtype, datetime_data, datetime_as_string, \ - busday_offset, busdaydef + busday_offset, busday_count, busdaydef import types as _types import sys diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index 08b3072237b2..5e1dec89d29b 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -22,6 +22,21 @@ #include "datetime_busday.h" #include "datetime_busdaydef.h" +/* Gets the day of the week for a datetime64[D] value */ +static int +get_day_of_week(npy_datetime date) +{ + int day_of_week; + + /* Get the day of the week for 'date' (1970-01-05 is Monday) */ + day_of_week = (int)((date - 4) % 7); + if (day_of_week < 0) { + day_of_week += 7; + } + + return day_of_week; +} + /* * Returns 1 if the date is a holiday (contained in the sorted * list of dates), 0 otherwise. @@ -154,11 +169,8 @@ apply_business_day_roll(npy_datetime date, npy_datetime *out, } } - /* Get the day of the week for 'date' (1970-01-05 is Monday) */ - day_of_week = (int)((date - 4) % 7); - if (day_of_week < 0) { - day_of_week += 7; - } + /* Get the day of the week for 'date' */ + day_of_week = get_day_of_week(date); /* Apply the 'roll' if it's not a business day */ if (weekmask[day_of_week] == 0 || @@ -331,6 +343,70 @@ apply_business_day_offset(npy_datetime date, npy_int64 offset, return 0; } +/* + * Applies a single business day count operation. See the function + * business_day_count for the meaning of all the parameters. + * + * Returns 0 on success, -1 on failure. + */ +static int +apply_business_day_count(npy_datetime date_begin, npy_datetime date_end, + npy_int64 *out, + npy_bool *weekmask, int busdays_in_weekmask, + npy_datetime *holidays_begin, npy_datetime *holidays_end) +{ + npy_int64 count, whole_weeks; + int day_of_week = 0; + + /* If we get a NaT, raise an error */ + if (date_begin == NPY_DATETIME_NAT || date_end == NPY_DATETIME_NAT) { + PyErr_SetString(PyExc_ValueError, + "Cannot compute a business day count with a NaT (not-a-time) " + "date"); + return -1; + } + + /* Trivial empty date range */ + if (date_begin >= date_end) { + *out = 0; + return 0; + } + + /* Remove any earlier holidays */ + holidays_begin = find_earliest_holiday_on_or_after(date_begin, + holidays_begin, holidays_end); + /* Remove any later holidays */ + holidays_end = find_earliest_holiday_on_or_after(date_end, + holidays_begin, holidays_end); + + /* Start the count as negative the number of holidays in the range */ + count = -(holidays_end - holidays_begin); + + /* Add the whole weeks between date_begin and date_end */ + whole_weeks = (date_end - date_begin) / 7; + count += whole_weeks * busdays_in_weekmask; + date_begin += whole_weeks * 7; + + if (date_begin < date_end) { + /* Get the day of the week for 'date_begin' */ + day_of_week = get_day_of_week(date_begin); + + /* Count the remaining days one by one */ + while (date_begin < date_end) { + if (weekmask[day_of_week]) { + count++; + } + ++date_begin; + if (++day_of_week == 7) { + day_of_week = 0; + } + } + } + + *out = count; + return 0; +} + /* * Applies the given offsets in business days to the dates provided. * This is the low-level function which requires already cleaned input @@ -475,13 +551,137 @@ business_day_offset(PyArrayObject *dates, PyArrayObject *offsets, return ret; } -NPY_NO_EXPORT npy_intp +/* + * Counts the number of business days between two dates, not including + * the end date. This is the low-level function which requires already + * cleaned input data. + * + * dates_begin: An array of dates with 'datetime64[D]' data type. + * dates_end: An array of dates with 'datetime64[D]' data type. + * out: Either NULL, or an array with 'int64' data type + * in which to place the resulting dates. + * weekmask: A 7-element boolean mask, 1 for possible business days and 0 + * for non-business days. + * busdays_in_weekmask: A count of how many 1's there are in weekmask. + * holidays_begin/holidays_end: A sorted list of dates matching '[D]' + * unit metadata, with any dates falling on a day of the + * week without weekmask[i] == 1 already filtered out. + */ +NPY_NO_EXPORT PyArrayObject * business_day_count(PyArrayObject *dates_begin, PyArrayObject *dates_end, PyArrayObject *out, - NPY_BUSDAY_ROLL roll, npy_bool *weekmask, int busdays_in_weekmask, npy_datetime *holidays_begin, npy_datetime *holidays_end) { + PyArray_DatetimeMetaData temp_meta; + PyArray_Descr *dtypes[3] = {NULL, NULL, NULL}; + + NpyIter *iter = NULL; + PyArrayObject *op[3] = {NULL, NULL, NULL}; + npy_uint32 op_flags[3], flags; + + PyArrayObject *ret = NULL; + + if (busdays_in_weekmask == 0) { + PyErr_SetString(PyExc_ValueError, + "the business day weekmask must have at least one " + "valid business day"); + return -1; + } + + /* First create the data types for the dates and the int64 output */ + temp_meta.base = NPY_FR_D; + temp_meta.num = 1; + temp_meta.events = 1; + dtypes[0] = create_datetime_dtype(NPY_DATETIME, &temp_meta); + if (dtypes[0] == NULL) { + goto fail; + } + dtypes[1] = dtypes[0]; + Py_INCREF(dtypes[1]); + dtypes[2] = PyArray_DescrFromType(NPY_INT64); + if (dtypes[2] == NULL) { + goto fail; + } + + /* Set up the iterator parameters */ + flags = NPY_ITER_EXTERNAL_LOOP| + NPY_ITER_BUFFERED| + NPY_ITER_ZEROSIZE_OK; + op[0] = dates_begin; + op_flags[0] = NPY_ITER_READONLY | NPY_ITER_ALIGNED; + op[1] = dates_end; + op_flags[1] = NPY_ITER_READONLY | NPY_ITER_ALIGNED; + op[2] = out; + op_flags[2] = NPY_ITER_WRITEONLY | NPY_ITER_ALLOCATE | NPY_ITER_ALIGNED; + + /* Allocate the iterator */ + iter = NpyIter_MultiNew(3, op, flags, NPY_KEEPORDER, NPY_SAFE_CASTING, + op_flags, dtypes); + if (iter == NULL) { + goto fail; + } + + /* Loop over all elements */ + if (NpyIter_GetIterSize(iter) > 0) { + NpyIter_IterNextFunc *iternext; + char **dataptr; + npy_intp *strideptr, *innersizeptr; + + iternext = NpyIter_GetIterNext(iter, NULL); + if (iternext == NULL) { + goto fail; + } + dataptr = NpyIter_GetDataPtrArray(iter); + strideptr = NpyIter_GetInnerStrideArray(iter); + innersizeptr = NpyIter_GetInnerLoopSizePtr(iter); + + do { + char *data_dates_begin = dataptr[0]; + char *data_dates_end = dataptr[1]; + char *data_out = dataptr[2]; + npy_intp stride_dates_begin = strideptr[0]; + npy_intp stride_dates_end = strideptr[1]; + npy_intp stride_out = strideptr[2]; + npy_intp count = *innersizeptr; + + while (count--) { + if (apply_business_day_count(*(npy_int64 *)data_dates_begin, + *(npy_int64 *)data_dates_end, + (npy_int64 *)data_out, + weekmask, busdays_in_weekmask, + holidays_begin, holidays_end) < 0) { + goto fail; + } + + data_dates_begin += stride_dates_begin; + data_dates_end += stride_dates_end; + data_out += stride_out; + } + } while (iternext(iter)); + } + + /* Get the return object from the iterator */ + ret = NpyIter_GetOperandArray(iter)[2]; + Py_INCREF(ret); + + goto finish; + +fail: + Py_XDECREF(ret); + ret = NULL; + +finish: + Py_XDECREF(dtypes[0]); + Py_XDECREF(dtypes[1]); + Py_XDECREF(dtypes[2]); + if (iter != NULL) { + if (NpyIter_Deallocate(iter) != NPY_SUCCEED) { + Py_XDECREF(ret); + ret = NULL; + } + } + return ret; } static int @@ -710,3 +910,147 @@ array_busday_offset(PyObject *NPY_UNUSED(self), return NULL; } + +/* + * This is the 'busday_count' function exposed for calling + * from Python. + */ +NPY_NO_EXPORT PyObject * +array_busday_count(PyObject *NPY_UNUSED(self), + PyObject *args, PyObject *kwds) +{ + char *kwlist[] = {"dates_begin", "dates_end", + "weekmask", "holidays", "busdaydef", "out", NULL}; + + PyObject *dates_begin_in = NULL, *dates_end_in = NULL, *out_in = NULL; + + PyArrayObject *dates_begin = NULL, *dates_end = NULL, *out = NULL, *ret; + npy_bool weekmask[7] = {2, 1, 1, 1, 1, 0, 0}; + NpyBusinessDayDef *busdaydef = NULL; + int i, busdays_in_weekmask; + npy_holidayslist holidays = {NULL, NULL}; + int allocated_holidays = 1; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "OO|O&O&O!O:busday_count", kwlist, + &dates_begin_in, + &dates_end_in, + &PyArray_WeekMaskConverter, &weekmask[0], + &PyArray_HolidaysConverter, &holidays, + &NpyBusinessDayDef_Type, &busdaydef, + &out_in)) { + goto fail; + } + + /* Make sure only one of the weekmask/holidays and busdaydef is supplied */ + if (busdaydef != NULL) { + if (weekmask[0] != 2 || holidays.begin != NULL) { + PyErr_SetString(PyExc_ValueError, + "Cannot supply both the weekmask/holidays and the " + "busdaydef parameters to busday_count()"); + goto fail; + } + + /* Indicate that the holidays weren't allocated by us */ + allocated_holidays = 0; + + /* Copy the private normalized weekmask/holidays data */ + holidays = busdaydef->holidays; + busdays_in_weekmask = busdaydef->busdays_in_weekmask; + memcpy(weekmask, busdaydef->weekmask, 7); + } + else { + /* + * Fix up the weekmask from the uninitialized + * signal value to a proper default. + */ + if (weekmask[0] == 2) { + weekmask[0] = 1; + } + + /* Count the number of business days in a week */ + busdays_in_weekmask = 0; + for (i = 0; i < 7; ++i) { + busdays_in_weekmask += weekmask[i]; + } + + /* The holidays list must be normalized before using it */ + normalize_holidays_list(&holidays, weekmask); + } + + /* Make 'dates_begin' into an array */ + if (PyArray_Check(dates_begin_in)) { + dates_begin = (PyArrayObject *)dates_begin_in; + Py_INCREF(dates_begin); + } + else { + PyArray_Descr *datetime_dtype; + + /* Use the datetime dtype with generic units so it fills it in */ + datetime_dtype = PyArray_DescrFromType(NPY_DATETIME); + if (datetime_dtype == NULL) { + goto fail; + } + + /* This steals the datetime_dtype reference */ + dates_begin = (PyArrayObject *)PyArray_FromAny(dates_begin_in, + datetime_dtype, + 0, 0, 0, dates_begin_in); + if (dates_begin == NULL) { + goto fail; + } + } + + /* Make 'dates_end' into an array */ + if (PyArray_Check(dates_end_in)) { + dates_end = (PyArrayObject *)dates_end_in; + Py_INCREF(dates_end); + } + else { + PyArray_Descr *datetime_dtype; + + /* Use the datetime dtype with generic units so it fills it in */ + datetime_dtype = PyArray_DescrFromType(NPY_DATETIME); + if (datetime_dtype == NULL) { + goto fail; + } + + /* This steals the datetime_dtype reference */ + dates_end = (PyArrayObject *)PyArray_FromAny(dates_end_in, + datetime_dtype, + 0, 0, 0, dates_end_in); + if (dates_end == NULL) { + goto fail; + } + } + + /* Make sure 'out' is an array if it's provided */ + if (out_in != NULL) { + if (!PyArray_Check(out_in)) { + PyErr_SetString(PyExc_ValueError, + "busday_offset: must provide a NumPy array for 'out'"); + goto fail; + } + out = (PyArrayObject *)out_in; + } + + ret = business_day_count(dates_begin, dates_end, out, + weekmask, busdays_in_weekmask, + holidays.begin, holidays.end); + + Py_DECREF(dates_begin); + Py_DECREF(dates_end); + if (allocated_holidays && holidays.begin != NULL) { + PyArray_free(holidays.begin); + } + + return out == NULL ? PyArray_Return(ret) : (PyObject *)ret; +fail: + Py_XDECREF(dates_begin); + Py_XDECREF(dates_end); + if (allocated_holidays && holidays.begin != NULL) { + PyArray_free(holidays.begin); + } + + return NULL; +} diff --git a/numpy/core/src/multiarray/datetime_busday.h b/numpy/core/src/multiarray/datetime_busday.h index f1cb68577f34..5d817b255487 100644 --- a/numpy/core/src/multiarray/datetime_busday.h +++ b/numpy/core/src/multiarray/datetime_busday.h @@ -2,22 +2,19 @@ #define _NPY_PRIVATE__DATETIME_BUSDAY_H_ /* - * A list of holidays, which should sorted, not contain any - * duplicates or NaTs, and not include any days already excluded - * by the associated weekmask. - * - * The data is manually managed with PyArray_malloc/PyArray_free. + * This is the 'busday_offset' function exposed for calling + * from Python. */ -typedef struct { - npy_datetime *begin, *end; -} npy_holidayslist; +NPY_NO_EXPORT PyObject * +array_busday_offset(PyObject *NPY_UNUSED(self), + PyObject *args, PyObject *kwds); /* - * This is the 'busday_offset' function exposed for calling + * This is the 'busday_count' function exposed for calling * from Python. */ NPY_NO_EXPORT PyObject * -array_busday_offset(PyObject *NPY_UNUSED(self), +array_busday_count(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds); diff --git a/numpy/core/src/multiarray/datetime_busdaydef.h b/numpy/core/src/multiarray/datetime_busdaydef.h index 64e5244a8682..b5f100910e91 100644 --- a/numpy/core/src/multiarray/datetime_busdaydef.h +++ b/numpy/core/src/multiarray/datetime_busdaydef.h @@ -1,6 +1,24 @@ #ifndef _NPY_PRIVATE__DATETIME_BUSDAYDEF_H_ #define _NPY_PRIVATE__DATETIME_BUSDAYDEF_H_ +/* + * A list of holidays, which should be sorted, not contain any + * duplicates or NaTs, and not include any days already excluded + * by the associated weekmask. + * + * The data is manually managed with PyArray_malloc/PyArray_free. + */ +typedef struct { + npy_datetime *begin, *end; +} npy_holidayslist; + +/* + * This object encapsulates a weekmask and normalized holidays list, + * so that the business day API can use this data without having + * to normalize it repeatedly. All the data of this object is private + * and cannot be modified from Python. Copies are made when giving + * the weekmask and holidays data to Python code. + */ typedef struct { PyObject_HEAD npy_holidayslist holidays; diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 5b973bc8d154..c515b0a82d2b 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -3603,6 +3603,9 @@ static struct PyMethodDef array_module_methods[] = { {"busday_offset", (PyCFunction)array_busday_offset, METH_VARARGS | METH_KEYWORDS, NULL}, + {"busday_count", + (PyCFunction)array_busday_count, + METH_VARARGS | METH_KEYWORDS, NULL}, #if !defined(NPY_PY3K) {"newbuffer", (PyCFunction)new_buffer, diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index ecac60e5dd37..28f53e3f0b8c 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -1388,6 +1388,31 @@ def test_datetime_busday_holidays_offset(self): '2012-03-05', '2012-03-07', '2012-03-06']), np.datetime64('2012-03-08')) + def test_datetime_busday_holidays_count(self): + holidays=['2011-01-01', '2011-10-10', '2011-11-11', '2011-11-24', + '2011-12-25', '2011-05-30', '2011-02-21', '2011-01-17', + '2011-12-26', '2012-01-02', '2011-02-21', '2011-05-30', + '2011-07-01', '2011-07-04', '2011-09-05', '2011-10-10'] + bdd = np.busdaydef(weekmask='1111100', holidays=holidays) + + # Validate against busday_offset broadcast against + # a range of offsets + dates = np.busday_offset('2011-01-01', np.arange(366), + roll='forward', busdaydef=bdd) + assert_equal(np.busday_count('2011-01-01', dates, busdaydef=bdd), + np.arange(366)) + + dates = np.busday_offset('2011-12-31', -np.arange(366), + roll='forward', busdaydef=bdd) + assert_equal(np.busday_count(dates, '2011-12-31', busdaydef=bdd), + np.arange(366)) + + # Can't supply both a weekmask/holidays and busdaydef + assert_raises(ValueError, np.busday_offset, '2012-01-03', '2012-02-03', + weekmask='1111100', busdaydef=bdd) + assert_raises(ValueError, np.busday_offset, '2012-01-03', '2012-02-03', + holidays=holidays, busdaydef=bdd) + class TestDateTimeData(TestCase): From 84e1f7d8415679231944dec9f5eaa182e0a9879c Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Tue, 14 Jun 2011 11:21:57 -0500 Subject: [PATCH 24/34] ENH: datetime-bday: Got is_busday function working, completed business day API --- numpy/core/numerictypes.py | 4 +- numpy/core/src/multiarray/datetime_busday.c | 247 ++++++++++++++++++- numpy/core/src/multiarray/datetime_busday.h | 7 + numpy/core/src/multiarray/multiarraymodule.c | 3 + numpy/core/tests/test_datetime.py | 19 ++ 5 files changed, 277 insertions(+), 3 deletions(-) diff --git a/numpy/core/numerictypes.py b/numpy/core/numerictypes.py index 0b61b7161663..8d184e47d3a9 100644 --- a/numpy/core/numerictypes.py +++ b/numpy/core/numerictypes.py @@ -93,11 +93,11 @@ 'ScalarType', 'obj2sctype', 'cast', 'nbytes', 'sctype2char', 'maximum_sctype', 'issctype', 'typecodes', 'find_common_type', 'issubdtype', 'datetime_data','datetime_as_string', - 'busday_offset', 'busday_count', 'busdaydef'] + 'busday_offset', 'busday_count', 'is_busday', 'busdaydef'] from numpy.core.multiarray import typeinfo, ndarray, array, \ empty, dtype, datetime_data, datetime_as_string, \ - busday_offset, busday_count, busdaydef + busday_offset, busday_count, is_busday, busdaydef import types as _types import sys diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index 5e1dec89d29b..57841f938152 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -586,7 +586,7 @@ business_day_count(PyArrayObject *dates_begin, PyArrayObject *dates_end, PyErr_SetString(PyExc_ValueError, "the business day weekmask must have at least one " "valid business day"); - return -1; + return NULL; } /* First create the data types for the dates and the int64 output */ @@ -684,6 +684,133 @@ business_day_count(PyArrayObject *dates_begin, PyArrayObject *dates_end, return ret; } +/* + * Returns a boolean array with True for input dates which are valid + * business days, and False for dates which are not. This is the + * low-level function which requires already cleaned input data. + * + * dates: An array of dates with 'datetime64[D]' data type. + * out: Either NULL, or an array with 'bool' data type + * in which to place the resulting dates. + * weekmask: A 7-element boolean mask, 1 for possible business days and 0 + * for non-business days. + * busdays_in_weekmask: A count of how many 1's there are in weekmask. + * holidays_begin/holidays_end: A sorted list of dates matching '[D]' + * unit metadata, with any dates falling on a day of the + * week without weekmask[i] == 1 already filtered out. + */ +NPY_NO_EXPORT PyArrayObject * +is_business_day(PyArrayObject *dates, PyArrayObject *out, + npy_bool *weekmask, int busdays_in_weekmask, + npy_datetime *holidays_begin, npy_datetime *holidays_end) +{ + PyArray_DatetimeMetaData temp_meta; + PyArray_Descr *dtypes[2] = {NULL, NULL}; + + NpyIter *iter = NULL; + PyArrayObject *op[2] = {NULL, NULL}; + npy_uint32 op_flags[2], flags; + + PyArrayObject *ret = NULL; + + if (busdays_in_weekmask == 0) { + PyErr_SetString(PyExc_ValueError, + "the business day weekmask must have at least one " + "valid business day"); + return NULL; + } + + /* First create the data types for the dates and the bool output */ + temp_meta.base = NPY_FR_D; + temp_meta.num = 1; + temp_meta.events = 1; + dtypes[0] = create_datetime_dtype(NPY_DATETIME, &temp_meta); + if (dtypes[0] == NULL) { + goto fail; + } + dtypes[1] = PyArray_DescrFromType(NPY_BOOL); + if (dtypes[1] == NULL) { + goto fail; + } + + /* Set up the iterator parameters */ + flags = NPY_ITER_EXTERNAL_LOOP| + NPY_ITER_BUFFERED| + NPY_ITER_ZEROSIZE_OK; + op[0] = dates; + op_flags[0] = NPY_ITER_READONLY | NPY_ITER_ALIGNED; + op[1] = out; + op_flags[1] = NPY_ITER_WRITEONLY | NPY_ITER_ALLOCATE | NPY_ITER_ALIGNED; + + /* Allocate the iterator */ + iter = NpyIter_MultiNew(2, op, flags, NPY_KEEPORDER, NPY_SAFE_CASTING, + op_flags, dtypes); + if (iter == NULL) { + goto fail; + } + + /* Loop over all elements */ + if (NpyIter_GetIterSize(iter) > 0) { + NpyIter_IterNextFunc *iternext; + char **dataptr; + npy_intp *strideptr, *innersizeptr; + + iternext = NpyIter_GetIterNext(iter, NULL); + if (iternext == NULL) { + goto fail; + } + dataptr = NpyIter_GetDataPtrArray(iter); + strideptr = NpyIter_GetInnerStrideArray(iter); + innersizeptr = NpyIter_GetInnerLoopSizePtr(iter); + + do { + char *data_dates = dataptr[0]; + char *data_out = dataptr[1]; + npy_intp stride_dates = strideptr[0]; + npy_intp stride_out = strideptr[1]; + npy_intp count = *innersizeptr; + + npy_datetime date; + int day_of_week; + + while (count--) { + /* Check if it's a business day */ + date = *(npy_datetime *)data_dates; + day_of_week = get_day_of_week(date); + *(npy_bool *)data_out = weekmask[day_of_week] && + !is_holiday(date, + holidays_begin, holidays_end) && + date != NPY_DATETIME_NAT; + + + data_dates += stride_dates; + data_out += stride_out; + } + } while (iternext(iter)); + } + + /* Get the return object from the iterator */ + ret = NpyIter_GetOperandArray(iter)[1]; + Py_INCREF(ret); + + goto finish; + +fail: + Py_XDECREF(ret); + ret = NULL; + +finish: + Py_XDECREF(dtypes[0]); + Py_XDECREF(dtypes[1]); + if (iter != NULL) { + if (NpyIter_Deallocate(iter) != NPY_SUCCEED) { + Py_XDECREF(ret); + ret = NULL; + } + } + return ret; +} + static int PyArray_BusDayRollConverter(PyObject *roll_in, NPY_BUSDAY_ROLL *roll) { @@ -1054,3 +1181,121 @@ array_busday_count(PyObject *NPY_UNUSED(self), return NULL; } + +/* + * This is the 'is_busday' function exposed for calling + * from Python. + */ +NPY_NO_EXPORT PyObject * +array_is_busday(PyObject *NPY_UNUSED(self), + PyObject *args, PyObject *kwds) +{ + char *kwlist[] = {"dates", + "weekmask", "holidays", "busdaydef", "out", NULL}; + + PyObject *dates_in = NULL, *out_in = NULL; + + PyArrayObject *dates = NULL,*out = NULL, *ret; + npy_bool weekmask[7] = {2, 1, 1, 1, 1, 0, 0}; + NpyBusinessDayDef *busdaydef = NULL; + int i, busdays_in_weekmask; + npy_holidayslist holidays = {NULL, NULL}; + int allocated_holidays = 1; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O|O&O&O!O:busday_count", kwlist, + &dates_in, + &PyArray_WeekMaskConverter, &weekmask[0], + &PyArray_HolidaysConverter, &holidays, + &NpyBusinessDayDef_Type, &busdaydef, + &out_in)) { + goto fail; + } + + /* Make sure only one of the weekmask/holidays and busdaydef is supplied */ + if (busdaydef != NULL) { + if (weekmask[0] != 2 || holidays.begin != NULL) { + PyErr_SetString(PyExc_ValueError, + "Cannot supply both the weekmask/holidays and the " + "busdaydef parameters to is_busday()"); + goto fail; + } + + /* Indicate that the holidays weren't allocated by us */ + allocated_holidays = 0; + + /* Copy the private normalized weekmask/holidays data */ + holidays = busdaydef->holidays; + busdays_in_weekmask = busdaydef->busdays_in_weekmask; + memcpy(weekmask, busdaydef->weekmask, 7); + } + else { + /* + * Fix up the weekmask from the uninitialized + * signal value to a proper default. + */ + if (weekmask[0] == 2) { + weekmask[0] = 1; + } + + /* Count the number of business days in a week */ + busdays_in_weekmask = 0; + for (i = 0; i < 7; ++i) { + busdays_in_weekmask += weekmask[i]; + } + + /* The holidays list must be normalized before using it */ + normalize_holidays_list(&holidays, weekmask); + } + + /* Make 'dates' into an array */ + if (PyArray_Check(dates_in)) { + dates = (PyArrayObject *)dates_in; + Py_INCREF(dates); + } + else { + PyArray_Descr *datetime_dtype; + + /* Use the datetime dtype with generic units so it fills it in */ + datetime_dtype = PyArray_DescrFromType(NPY_DATETIME); + if (datetime_dtype == NULL) { + goto fail; + } + + /* This steals the datetime_dtype reference */ + dates = (PyArrayObject *)PyArray_FromAny(dates_in, + datetime_dtype, + 0, 0, 0, dates_in); + if (dates == NULL) { + goto fail; + } + } + + /* Make sure 'out' is an array if it's provided */ + if (out_in != NULL) { + if (!PyArray_Check(out_in)) { + PyErr_SetString(PyExc_ValueError, + "busday_offset: must provide a NumPy array for 'out'"); + goto fail; + } + out = (PyArrayObject *)out_in; + } + + ret = is_business_day(dates, out, + weekmask, busdays_in_weekmask, + holidays.begin, holidays.end); + + Py_DECREF(dates); + if (allocated_holidays && holidays.begin != NULL) { + PyArray_free(holidays.begin); + } + + return out == NULL ? PyArray_Return(ret) : (PyObject *)ret; +fail: + Py_XDECREF(dates); + if (allocated_holidays && holidays.begin != NULL) { + PyArray_free(holidays.begin); + } + + return NULL; +} diff --git a/numpy/core/src/multiarray/datetime_busday.h b/numpy/core/src/multiarray/datetime_busday.h index 5d817b255487..483151122b2a 100644 --- a/numpy/core/src/multiarray/datetime_busday.h +++ b/numpy/core/src/multiarray/datetime_busday.h @@ -17,5 +17,12 @@ NPY_NO_EXPORT PyObject * array_busday_count(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds); +/* + * This is the 'is_busday' function exposed for calling + * from Python. + */ +NPY_NO_EXPORT PyObject * +array_is_busday(PyObject *NPY_UNUSED(self), + PyObject *args, PyObject *kwds); #endif diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index c515b0a82d2b..60ee9dbe8032 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -3606,6 +3606,9 @@ static struct PyMethodDef array_module_methods[] = { {"busday_count", (PyCFunction)array_busday_count, METH_VARARGS | METH_KEYWORDS, NULL}, + {"is_busday", + (PyCFunction)array_is_busday, + METH_VARARGS | METH_KEYWORDS, NULL}, #if !defined(NPY_PY3K) {"newbuffer", (PyCFunction)new_buffer, diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index 28f53e3f0b8c..a3adcb2a6af6 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -1412,7 +1412,26 @@ def test_datetime_busday_holidays_count(self): weekmask='1111100', busdaydef=bdd) assert_raises(ValueError, np.busday_offset, '2012-01-03', '2012-02-03', holidays=holidays, busdaydef=bdd) + + # Number of Mondays in March 2011 + assert_equal(np.busday_count('2011-03', '2011-04', weekmask='Mon'), 4) + def test_datetime_is_busday(self): + holidays=['2011-01-01', '2011-10-10', '2011-11-11', '2011-11-24', + '2011-12-25', '2011-05-30', '2011-02-21', '2011-01-17', + '2011-12-26', '2012-01-02', '2011-02-21', '2011-05-30', + '2011-07-01', '2011-07-04', '2011-09-05', '2011-10-10', + 'NaT'] + bdd = np.busdaydef(weekmask='1111100', holidays=holidays) + + # Weekend/weekday tests + assert_equal(np.is_busday('2011-01-01'), False) + assert_equal(np.is_busday('2011-01-02'), False) + assert_equal(np.is_busday('2011-01-03'), True) + + # All the holidays are not business days + assert_equal(np.is_busday(holidays, busdaydef=bdd), + np.zeros(len(holidays), dtype='?')) class TestDateTimeData(TestCase): From 6b5a42a777b16812e774193b06da1b68b92bc689 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Tue, 14 Jun 2011 15:28:59 -0500 Subject: [PATCH 25/34] DOC: datetime-bday: Document the datetime business day functions --- doc/source/reference/routines.datetime.rst | 20 ++ doc/source/reference/routines.rst | 1 + numpy/add_newdocs.py | 262 ++++++++++++++++++++ numpy/core/src/multiarray/datetime_busday.c | 5 +- 4 files changed, 286 insertions(+), 2 deletions(-) create mode 100644 doc/source/reference/routines.datetime.rst diff --git a/doc/source/reference/routines.datetime.rst b/doc/source/reference/routines.datetime.rst new file mode 100644 index 000000000000..d8517bd396ae --- /dev/null +++ b/doc/source/reference/routines.datetime.rst @@ -0,0 +1,20 @@ +.. _routines.datetime: + +Datetime Support Functions +************************** + +.. currentmodule:: numpy + +Business Day Functions +====================== + +.. currentmodule:: numpy + +.. autosummary:: + :toctree: generated/ + + busdaydef + is_busday + busday_offset + busday_count + diff --git a/doc/source/reference/routines.rst b/doc/source/reference/routines.rst index 0788d3a0ab85..c97a3d244fc2 100644 --- a/doc/source/reference/routines.rst +++ b/doc/source/reference/routines.rst @@ -30,6 +30,7 @@ indentation. routines.math routines.functional routines.poly + routines.datetime routines.financial routines.set routines.window diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py index 20ce6c37804b..49c0811bd602 100644 --- a/numpy/add_newdocs.py +++ b/numpy/add_newdocs.py @@ -5934,6 +5934,268 @@ def luf(lamdaexpr, *args, **kwargs): """)) +############################################################################## +# +# Datetime-related Methods +# +############################################################################## + +add_newdoc('numpy.core.multiarray', 'busdaydef', + """ + busdaydef(weekmask='1111100', holidays=None) + + An object that efficiently stores information defining business + days for the business day-related functions. + + Parameters + ---------- + weekmask : string or array_like of bool + A seven-element array indicating which of Monday through Sunday may + be valid business days. May be specified as a list or array, like + [1,1,1,1,1,0,0], a length-seven string like '1111100', or a string + of three-letter weekday names, like 'MonTueWedThuFri'. The latter + string representation is most useful when only one day of the + week is important, like 'Mon' if you want to calculate the date + of Easter. + holidays : array_like of datetime64[D] + An array of dates which should be blacked out from being considered + as business days. They may be specified in any order, and NaT + (not-a-time) dates are ignored. Internally, this list is normalized + into a form suited for fast business day calculations. + + Returns + ------- + out : busdaydef + A business day definition object containing the specified + weekmask and holidays. + + See Also + -------- + is_busday : Returns a boolean array indicating valid business days. + busday_offset : Applies an offset counted in business days. + busday_count : Counts how many business days are in a half-open date range. + + Attributes + ---------- + weekmask : seven-element array of bool + holidays : sorted array of datetime64[D] + + Examples + -------- + >>> # Some important days in July + ... bdd = np.busdaydef(holidays=['2011-07-01', '2011-07-04', '2011-07-17']) + >>> # Default is Monday to Friday weekdays + ... bdd.weekmask + array([ True, True, True, True, True, False, False], dtype='bool') + >>> # Any holidays already on the weekend are removed + ... bdd.holidays + array(['2011-07-01', '2011-07-04'], dtype='datetime64[D]') + """) + +add_newdoc('numpy.core.multiarray', 'busdaydef', ('weekmask', + """A copy of the seven-element boolean mask indicating valid business days.""")) + +add_newdoc('numpy.core.multiarray', 'busdaydef', ('holidays', + """A copy of the holiday array indicating blacked out business days.""")) + +add_newdoc('numpy.core.multiarray', 'is_busday', + """ + is_busday(dates, weekmask='1111100', holidays=None, busdaydef=None, out=None) + + Calculates which of the given dates are valid business days, and + which are not. + + Parameters + ---------- + dates : array_like of datetime64[D] + The array of dates to process. + weekmask : string or array_like of bool + A seven-element array indicating which of Monday through Sunday may + be valid business days. May be specified as a list or array, like + [1,1,1,1,1,0,0], a length-seven string like '1111100', or a string + of three-letter weekday names, like 'MonTueWedThuFri'. The latter + string representation is most useful when only one day of the + week is important, like 'Mon' if you want to calculate the date + of Easter. + holidays : array_like of datetime64[D] + An array of dates which should be blacked out from being considered + as business days. They may be specified in any order, and NaT + (not-a-time) dates are ignored. Internally, this list is normalized + into a form suited for fast business day calculations. + busdaydef : busdaydef + A `busdaydef` object which specifies the business days. If this + parameter is provided, neither weekmask nor holidays may be + provided. + out : array of bool + If provided, this array is filled with the result. + + Returns + ------- + out : array of bool + An array containing True for each valid business day, and + False for the others. + + See Also + -------- + busdaydef: An object for efficiently specifying which are business days. + busday_offset : Applies an offset counted in business days. + busday_count : Counts how many business days are in a half-open date range. + + Examples + -------- + >>> # The weekdays are Friday, Saturday, and Monday + ... np.is_busday(['2011-07-01', '2011-07-02', '2011-07-18'], + ... holidays=['2011-07-01', '2011-07-04', '2011-07-17']) + array([False, False, True], dtype='bool') + """) + +add_newdoc('numpy.core.multiarray', 'busday_offset', + """ + busday_offset(dates, offsets, roll='raise', weekmask='1111100', holidays=None, busdaydef=None, out=None) + + First adjusts the date to fall on a business day according to + the ``roll`` rule, then applies offsets to the given dates + counted in business days. + + Parameters + ---------- + dates : array_like of datetime64[D] + The array of dates to process. + offsets : array_like of integer + The array of offsets, which is broadcast with ``dates``. + roll : {'raise', 'nat', 'forward', 'following', 'backward', 'preceding', 'modifiedfollowing', 'modifiedpreceding'} + How to treat dates that do not fall on a business day. The default + is 'raise'. + + * 'raise' means to raise an exception for invalid business days. + * 'nat' means to return a NaT (not-a-time) for invalid business days. + * 'forward' and 'following' mean to take the first business day + later in time. + * 'backward' and 'preceding' mean to take the first business day + earlier in time. + * 'modifiedfollowing' means to take the first business day + later in time unless it is across a Month boundary, in which + case to take the first business day earlier in time. + * 'modifiedpreceding' means to take the first business day + earlier in time unless it is across a Month boundary, in which + case to take the first business day later in time. + weekmask : string or array_like of bool + A seven-element array indicating which of Monday through Sunday may + be valid business days. May be specified as a list or array, like + [1,1,1,1,1,0,0], a length-seven string like '1111100', or a string + of three-letter weekday names, like 'MonTueWedThuFri'. The latter + string representation is most useful when only one day of the + week is important, like 'Mon' if you want to calculate the date + of Easter. + holidays : array_like of datetime64[D] + An array of dates which should be blacked out from being considered + as business days. They may be specified in any order, and NaT + (not-a-time) dates are ignored. Internally, this list is normalized + into a form suited for fast business day calculations. + busdaydef : busdaydef + A `busdaydef` object which specifies the business days. If this + parameter is provided, neither weekmask nor holidays may be + provided. + out : array of datetime64[D] + If provided, this array is filled with the result. + + Returns + ------- + out : array of datetime64[D] + An array containing the dates with offsets applied. + + See Also + -------- + busdaydef: An object for efficiently specifying which are business days. + is_busday : Returns a boolean array indicating valid business days. + busday_count : Counts how many business days are in a half-open date range. + + Examples + -------- + >>> # First business day in October 2011 (not accounting for holidays) + ... np.busday_offset('2011-10', 0, roll='forward') + numpy.datetime64('2011-10-03','D') + >>> # Last business day in February 2012 (not accounting for holidays) + ... np.busday_offset('2012-03', -1, roll='forward') + numpy.datetime64('2012-02-29','D') + >>> # Third Wednesday in January 2011 + ... np.busday_offset('2011-01', 2, roll='forward', weekmask='Wed') + numpy.datetime64('2011-01-19','D') + >>> # 2012 Mother's Day in Canada and the U.S. + ... np.busday_offset('2012-05', 1, roll='forward', weekmask='Sun') + numpy.datetime64('2012-05-13','D') + + >>> # First business day on or after a date + ... np.busday_offset('2011-03-20', 0, roll='forward') + numpy.datetime64('2011-03-21','D') + >>> np.busday_offset('2011-03-22', 0, roll='forward') + numpy.datetime64('2011-03-22','D') + >>> # First business day after a date + ... np.busday_offset('2011-03-20', 1, roll='backward') + numpy.datetime64('2011-03-21','D') + >>> np.busday_offset('2011-03-22', 1, roll='backward') + numpy.datetime64('2011-03-23','D') + """) + +add_newdoc('numpy.core.multiarray', 'busday_count', + """ + busday_count(begindates, enddates, weekmask='1111100', holidays=[], busdaydef=None, out=None) + + Counts the number of business days between `begindates` and + `enddates`, not including the day of `enddates`. + + Parameters + ---------- + begindates : array_like of datetime64[D] + The array of the first dates for counting. + enddates : array_like of datetime64[D] + The array of the end dates for counting, which are excluded + from the count themselves. + weekmask : string or array_like of bool + A seven-element array indicating which of Monday through Sunday may + be valid business days. May be specified as a list or array, like + [1,1,1,1,1,0,0], a length-seven string like '1111100', or a string + of three-letter weekday names, like 'MonTueWedThuFri'. The latter + string representation is most useful when only one day of the + week is important, like 'Mon' if you want to calculate the date + of Easter. + holidays : array_like of datetime64[D] + An array of dates which should be blacked out from being considered + as business days. They may be specified in any order, and NaT + (not-a-time) dates are ignored. Internally, this list is normalized + into a form suited for fast business day calculations. + busdaydef : busdaydef + A `busdaydef` object which specifies the business days. If this + parameter is provided, neither weekmask nor holidays may be + provided. + out : array of int64 + If provided, this array is filled with the result. + + Returns + ------- + out : array of int64 + An array containing the number of business days between + the begin and end dates. + + See Also + -------- + busdaydef: An object for efficiently specifying which are business days. + is_busday : Returns a boolean array indicating valid business days. + busday_offset : Applies an offset counted in business days. + + Examples + -------- + >>> # Number of weekdays in January 2011 + ... np.busday_count('2011-01', '2011-02') + 21 + >>> # Number of weekdays in 2011 + ... np.busday_count('2011', '2012') + 260 + >>> # Number of Saturdays in 2011 + ... np.busday_count('2011', '2012', weekmask='Sat') + 53 + """) + ############################################################################## # # nd_grid instances diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index 57841f938152..728a7fa16a95 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -141,6 +141,7 @@ find_earliest_holiday_after(npy_datetime date, return holidays_begin; } + /* * Applies the 'roll' strategy to 'date', placing the result in 'out' * and setting 'out_day_of_week' to the day of the week that results. @@ -1046,7 +1047,7 @@ NPY_NO_EXPORT PyObject * array_busday_count(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) { - char *kwlist[] = {"dates_begin", "dates_end", + char *kwlist[] = {"begindates", "enddates", "weekmask", "holidays", "busdaydef", "out", NULL}; PyObject *dates_begin_in = NULL, *dates_end_in = NULL, *out_in = NULL; @@ -1203,7 +1204,7 @@ array_is_busday(PyObject *NPY_UNUSED(self), int allocated_holidays = 1; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|O&O&O!O:busday_count", kwlist, + "O|O&O&O!O:is_busday", kwlist, &dates_in, &PyArray_WeekMaskConverter, &weekmask[0], &PyArray_HolidaysConverter, &holidays, From 6376ba8fca4d9c71002b5ec59b4bb9813a95b63f Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Wed, 15 Jun 2011 10:34:54 -0500 Subject: [PATCH 26/34] DOC: datetime-feedback: Various comment/documentation tweaks from Chuck's review feedback --- numpy/add_newdocs.py | 10 ++++++---- numpy/core/src/multiarray/_datetime.h | 14 ++++---------- numpy/core/src/multiarray/ctors.c | 11 ++++++++++- numpy/core/src/multiarray/datetime.c | 6 +++--- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py index 49c0811bd602..270147ae211c 100644 --- a/numpy/add_newdocs.py +++ b/numpy/add_newdocs.py @@ -6032,8 +6032,8 @@ def luf(lamdaexpr, *args, **kwargs): Returns ------- out : array of bool - An array containing True for each valid business day, and - False for the others. + An array with the same shape as ``dates``, containing True for + each valid business day, and False for the others. See Also -------- @@ -6102,7 +6102,8 @@ def luf(lamdaexpr, *args, **kwargs): Returns ------- out : array of datetime64[D] - An array containing the dates with offsets applied. + An array with a shape from broadcasting ``dates`` and ``offsets`` + together, containing the dates with offsets applied. See Also -------- @@ -6174,7 +6175,8 @@ def luf(lamdaexpr, *args, **kwargs): Returns ------- out : array of int64 - An array containing the number of business days between + An array with a shape from broadcasting ``begindates`` and ``enddates`` + together, containing the number of business days between the begin and end dates. See Also diff --git a/numpy/core/src/multiarray/_datetime.h b/numpy/core/src/multiarray/_datetime.h index 12e833378e15..8a9ffff7b885 100644 --- a/numpy/core/src/multiarray/_datetime.h +++ b/numpy/core/src/multiarray/_datetime.h @@ -47,7 +47,10 @@ convert_datetimestruct_to_datetime(PyArray_DatetimeMetaData *meta, const npy_datetimestruct *dts, npy_datetime *out); -/* Extracts the month number from a 'datetime64[D]' value */ +/* + * Extracts the month number, within the current year, + * from a 'datetime64[D]' value. January is 1, etc. + */ NPY_NO_EXPORT int days_to_month_number(npy_datetime days); @@ -61,15 +64,6 @@ parse_datetime_metadata_from_metastr(char *metastr, Py_ssize_t len, PyArray_DatetimeMetaData *out_meta); -/* - * This function returns a reference to a capsule - * which contains the datetime metadata parsed from a metadata - * string. 'metastr' should be NULL-terminated, and len should - * contain its string length. - */ -NPY_NO_EXPORT PyObject * -parse_datetime_metacobj_from_metastr(char *metastr, Py_ssize_t len); - /* * Converts a datetype dtype string into a dtype descr object. * The "type" string should be NULL-terminated, and len should diff --git a/numpy/core/src/multiarray/ctors.c b/numpy/core/src/multiarray/ctors.c index 0ae7434819ce..8d4a7430b038 100644 --- a/numpy/core/src/multiarray/ctors.c +++ b/numpy/core/src/multiarray/ctors.c @@ -1690,7 +1690,16 @@ PyArray_FromAny(PyObject *op, PyArray_Descr *newtype, int min_depth, } } } - /* Treat datetime generic units with the same idea as flexible strings */ + /* + * Treat datetime generic units with the same idea as flexible strings. + * + * Flexible strings, for example the dtype 'str', use size zero as a + * signal indicating that they represent a "generic string type" instead + * of a string type with the size already baked in. The generic unit + * plays the same role, indicating that it's a "generic datetime type", + * and the actual unit should be filled in when needed just like the + * actual string size should be filled in when needed. + */ else if (newtype != NULL && newtype->type_num == NPY_DATETIME) { PyArray_DatetimeMetaData *meta = get_datetime_metadata_from_dtype(newtype); diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index a9f2f6749eb8..78a5032f5d62 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -1505,8 +1505,8 @@ get_datetime_conversion_factor(PyArray_DatetimeMetaData *src_meta, /* If something overflowed, make both num and denom 0 */ if (denom == 0) { PyErr_Format(PyExc_OverflowError, - "Integer overflow getting a conversion factor between " - "NumPy datetime unit %s and %s", + "Integer overflow while computing the conversion " + "factor between NumPy datetime units %s and %s", _datetime_strings[src_base], _datetime_strings[dst_base]); *out_num = 0; @@ -4660,7 +4660,7 @@ datetime_arange(PyObject *start, PyObject *stop, PyObject *step, } else { PyErr_SetString(PyExc_ValueError, - "arange: step may not be zero"); + "arange: step cannot be zero"); return NULL; } From d117335ca6f74015a773008ff4516270e1849f6b Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Wed, 15 Jun 2011 10:58:48 -0500 Subject: [PATCH 27/34] STY: datetime-feedback: Rename np.busdaydef -> np.busdaycalendar Also rename the busdaydef parameters to busdaycal parameters. This change was motivated by Chuck's code review feedback. --- doc/source/reference/routines.datetime.rst | 2 +- numpy/add_newdocs.py | 44 ++++++++------- numpy/core/SConscript | 2 +- numpy/core/code_generators/genapi.py | 2 +- numpy/core/numerictypes.py | 4 +- numpy/core/setup.py | 2 +- numpy/core/src/multiarray/datetime_busday.c | 56 +++++++++---------- ...etime_busdaydef.c => datetime_busdaycal.c} | 42 +++++++------- ...etime_busdaydef.h => datetime_busdaycal.h} | 7 +-- numpy/core/src/multiarray/multiarraymodule.c | 13 +++-- .../src/multiarray/multiarraymodule_onefile.c | 2 +- numpy/core/tests/test_datetime.py | 38 ++++++------- 12 files changed, 107 insertions(+), 107 deletions(-) rename numpy/core/src/multiarray/{datetime_busdaydef.c => datetime_busdaycal.c} (93%) rename numpy/core/src/multiarray/{datetime_busdaydef.h => datetime_busdaycal.h} (90%) diff --git a/doc/source/reference/routines.datetime.rst b/doc/source/reference/routines.datetime.rst index d8517bd396ae..aab6f1694d4c 100644 --- a/doc/source/reference/routines.datetime.rst +++ b/doc/source/reference/routines.datetime.rst @@ -13,7 +13,7 @@ Business Day Functions .. autosummary:: :toctree: generated/ - busdaydef + busdaycalendar is_busday busday_offset busday_count diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py index 270147ae211c..f1be98bd2857 100644 --- a/numpy/add_newdocs.py +++ b/numpy/add_newdocs.py @@ -5940,12 +5940,13 @@ def luf(lamdaexpr, *args, **kwargs): # ############################################################################## -add_newdoc('numpy.core.multiarray', 'busdaydef', +add_newdoc('numpy.core.multiarray', 'busdaycalendar', """ - busdaydef(weekmask='1111100', holidays=None) + busdaycalendar(weekmask='1111100', holidays=None) - An object that efficiently stores information defining business - days for the business day-related functions. + A business day calendar object that efficiently stores + information defining business days for the business + day-related functions. Parameters ---------- @@ -5965,8 +5966,8 @@ def luf(lamdaexpr, *args, **kwargs): Returns ------- - out : busdaydef - A business day definition object containing the specified + out : busdaycalendar + A business day calendar object containing the specified weekmask and holidays. See Also @@ -5983,7 +5984,8 @@ def luf(lamdaexpr, *args, **kwargs): Examples -------- >>> # Some important days in July - ... bdd = np.busdaydef(holidays=['2011-07-01', '2011-07-04', '2011-07-17']) + ... bdd = np.busdaycalendar( + ... holidays=['2011-07-01', '2011-07-04', '2011-07-17']) >>> # Default is Monday to Friday weekdays ... bdd.weekmask array([ True, True, True, True, True, False, False], dtype='bool') @@ -5992,15 +5994,15 @@ def luf(lamdaexpr, *args, **kwargs): array(['2011-07-01', '2011-07-04'], dtype='datetime64[D]') """) -add_newdoc('numpy.core.multiarray', 'busdaydef', ('weekmask', +add_newdoc('numpy.core.multiarray', 'busdaycalendar', ('weekmask', """A copy of the seven-element boolean mask indicating valid business days.""")) -add_newdoc('numpy.core.multiarray', 'busdaydef', ('holidays', +add_newdoc('numpy.core.multiarray', 'busdaycalendar', ('holidays', """A copy of the holiday array indicating blacked out business days.""")) add_newdoc('numpy.core.multiarray', 'is_busday', """ - is_busday(dates, weekmask='1111100', holidays=None, busdaydef=None, out=None) + is_busday(dates, weekmask='1111100', holidays=None, busdaycal=None, out=None) Calculates which of the given dates are valid business days, and which are not. @@ -6022,8 +6024,8 @@ def luf(lamdaexpr, *args, **kwargs): as business days. They may be specified in any order, and NaT (not-a-time) dates are ignored. Internally, this list is normalized into a form suited for fast business day calculations. - busdaydef : busdaydef - A `busdaydef` object which specifies the business days. If this + busdaycal : busdaycalendar + A `busdaycalendar` object which specifies the business days. If this parameter is provided, neither weekmask nor holidays may be provided. out : array of bool @@ -6037,7 +6039,7 @@ def luf(lamdaexpr, *args, **kwargs): See Also -------- - busdaydef: An object for efficiently specifying which are business days. + busdaycalendar: An object for efficiently specifying which are business days. busday_offset : Applies an offset counted in business days. busday_count : Counts how many business days are in a half-open date range. @@ -6051,7 +6053,7 @@ def luf(lamdaexpr, *args, **kwargs): add_newdoc('numpy.core.multiarray', 'busday_offset', """ - busday_offset(dates, offsets, roll='raise', weekmask='1111100', holidays=None, busdaydef=None, out=None) + busday_offset(dates, offsets, roll='raise', weekmask='1111100', holidays=None, busdaycal=None, out=None) First adjusts the date to fall on a business day according to the ``roll`` rule, then applies offsets to the given dates @@ -6092,8 +6094,8 @@ def luf(lamdaexpr, *args, **kwargs): as business days. They may be specified in any order, and NaT (not-a-time) dates are ignored. Internally, this list is normalized into a form suited for fast business day calculations. - busdaydef : busdaydef - A `busdaydef` object which specifies the business days. If this + busdaycal : busdaycalendar + A `busdaycalendar` object which specifies the business days. If this parameter is provided, neither weekmask nor holidays may be provided. out : array of datetime64[D] @@ -6107,7 +6109,7 @@ def luf(lamdaexpr, *args, **kwargs): See Also -------- - busdaydef: An object for efficiently specifying which are business days. + busdaycalendar: An object for efficiently specifying which are business days. is_busday : Returns a boolean array indicating valid business days. busday_count : Counts how many business days are in a half-open date range. @@ -6140,7 +6142,7 @@ def luf(lamdaexpr, *args, **kwargs): add_newdoc('numpy.core.multiarray', 'busday_count', """ - busday_count(begindates, enddates, weekmask='1111100', holidays=[], busdaydef=None, out=None) + busday_count(begindates, enddates, weekmask='1111100', holidays=[], busdaycal=None, out=None) Counts the number of business days between `begindates` and `enddates`, not including the day of `enddates`. @@ -6165,8 +6167,8 @@ def luf(lamdaexpr, *args, **kwargs): as business days. They may be specified in any order, and NaT (not-a-time) dates are ignored. Internally, this list is normalized into a form suited for fast business day calculations. - busdaydef : busdaydef - A `busdaydef` object which specifies the business days. If this + busdaycal : busdaycalendar + A `busdaycalendar` object which specifies the business days. If this parameter is provided, neither weekmask nor holidays may be provided. out : array of int64 @@ -6181,7 +6183,7 @@ def luf(lamdaexpr, *args, **kwargs): See Also -------- - busdaydef: An object for efficiently specifying which are business days. + busdaycalendar: An object for efficiently specifying which are business days. is_busday : Returns a boolean array indicating valid business days. busday_offset : Applies an offset counted in business days. diff --git a/numpy/core/SConscript b/numpy/core/SConscript index 37230e50af3c..0baea3d0c8ab 100644 --- a/numpy/core/SConscript +++ b/numpy/core/SConscript @@ -440,7 +440,7 @@ if ENABLE_SEPARATE_COMPILATION: pjoin('src', 'multiarray', 'arrayobject.c'), pjoin('src', 'multiarray', 'datetime.c'), pjoin('src', 'multiarray', 'datetime_busday.c'), - pjoin('src', 'multiarray', 'datetime_busdaydef.c'), + pjoin('src', 'multiarray', 'datetime_busdaycal.c'), pjoin('src', 'multiarray', 'numpyos.c'), pjoin('src', 'multiarray', 'flagsobject.c'), pjoin('src', 'multiarray', 'descriptor.c'), diff --git a/numpy/core/code_generators/genapi.py b/numpy/core/code_generators/genapi.py index 21e6444e6cb7..844aebff0af9 100644 --- a/numpy/core/code_generators/genapi.py +++ b/numpy/core/code_generators/genapi.py @@ -45,7 +45,7 @@ join('multiarray', 'buffer.c'), join('multiarray', 'datetime.c'), join('multiarray', 'datetime_busday.c'), - join('multiarray', 'datetime_busdaydef.c'), + join('multiarray', 'datetime_busdaycal.c'), join('multiarray', 'nditer.c.src'), join('multiarray', 'nditer_pywrap.c'), join('multiarray', 'einsum.c.src'), diff --git a/numpy/core/numerictypes.py b/numpy/core/numerictypes.py index 8d184e47d3a9..f1152cb74e33 100644 --- a/numpy/core/numerictypes.py +++ b/numpy/core/numerictypes.py @@ -93,11 +93,11 @@ 'ScalarType', 'obj2sctype', 'cast', 'nbytes', 'sctype2char', 'maximum_sctype', 'issctype', 'typecodes', 'find_common_type', 'issubdtype', 'datetime_data','datetime_as_string', - 'busday_offset', 'busday_count', 'is_busday', 'busdaydef'] + 'busday_offset', 'busday_count', 'is_busday', 'busdaycalendar'] from numpy.core.multiarray import typeinfo, ndarray, array, \ empty, dtype, datetime_data, datetime_as_string, \ - busday_offset, busday_count, is_busday, busdaydef + busday_offset, busday_count, is_busday, busdaycalendar import types as _types import sys diff --git a/numpy/core/setup.py b/numpy/core/setup.py index 0db4698ed187..87e4e4f90f0d 100644 --- a/numpy/core/setup.py +++ b/numpy/core/setup.py @@ -748,7 +748,7 @@ def get_mathlib_info(*args): join('src', 'multiarray', 'buffer.c'), join('src', 'multiarray', 'datetime.c'), join('src', 'multiarray', 'datetime_busday.c'), - join('src', 'multiarray', 'datetime_busdaydef.c'), + join('src', 'multiarray', 'datetime_busdaycal.c'), join('src', 'multiarray', 'numpyos.c'), join('src', 'multiarray', 'conversion_utils.c'), join('src', 'multiarray', 'flagsobject.c'), diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index 728a7fa16a95..034e7f6abc00 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -20,7 +20,7 @@ #include "lowlevel_strided_loops.h" #include "_datetime.h" #include "datetime_busday.h" -#include "datetime_busdaydef.h" +#include "datetime_busdaycal.h" /* Gets the day of the week for a datetime64[D] value */ static int @@ -918,14 +918,14 @@ array_busday_offset(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) { char *kwlist[] = {"dates", "offsets", "roll", - "weekmask", "holidays", "busdaydef", "out", NULL}; + "weekmask", "holidays", "busdaycal", "out", NULL}; PyObject *dates_in = NULL, *offsets_in = NULL, *out_in = NULL; PyArrayObject *dates = NULL, *offsets = NULL, *out = NULL, *ret; NPY_BUSDAY_ROLL roll = NPY_BUSDAY_RAISE; npy_bool weekmask[7] = {2, 1, 1, 1, 1, 0, 0}; - NpyBusinessDayDef *busdaydef = NULL; + NpyBusDayCalendar *busdaycal = NULL; int i, busdays_in_weekmask; npy_holidayslist holidays = {NULL, NULL}; int allocated_holidays = 1; @@ -937,17 +937,17 @@ array_busday_offset(PyObject *NPY_UNUSED(self), &PyArray_BusDayRollConverter, &roll, &PyArray_WeekMaskConverter, &weekmask[0], &PyArray_HolidaysConverter, &holidays, - &NpyBusinessDayDef_Type, &busdaydef, + &NpyBusDayCalendar_Type, &busdaycal, &out_in)) { goto fail; } - /* Make sure only one of the weekmask/holidays and busdaydef is supplied */ - if (busdaydef != NULL) { + /* Make sure only one of the weekmask/holidays and busdaycal is supplied */ + if (busdaycal != NULL) { if (weekmask[0] != 2 || holidays.begin != NULL) { PyErr_SetString(PyExc_ValueError, "Cannot supply both the weekmask/holidays and the " - "busdaydef parameters to busday_offset()"); + "busdaycal parameters to busday_offset()"); goto fail; } @@ -955,9 +955,9 @@ array_busday_offset(PyObject *NPY_UNUSED(self), allocated_holidays = 0; /* Copy the private normalized weekmask/holidays data */ - holidays = busdaydef->holidays; - busdays_in_weekmask = busdaydef->busdays_in_weekmask; - memcpy(weekmask, busdaydef->weekmask, 7); + holidays = busdaycal->holidays; + busdays_in_weekmask = busdaycal->busdays_in_weekmask; + memcpy(weekmask, busdaycal->weekmask, 7); } else { /* @@ -1048,13 +1048,13 @@ array_busday_count(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) { char *kwlist[] = {"begindates", "enddates", - "weekmask", "holidays", "busdaydef", "out", NULL}; + "weekmask", "holidays", "busdaycal", "out", NULL}; PyObject *dates_begin_in = NULL, *dates_end_in = NULL, *out_in = NULL; PyArrayObject *dates_begin = NULL, *dates_end = NULL, *out = NULL, *ret; npy_bool weekmask[7] = {2, 1, 1, 1, 1, 0, 0}; - NpyBusinessDayDef *busdaydef = NULL; + NpyBusDayCalendar *busdaycal = NULL; int i, busdays_in_weekmask; npy_holidayslist holidays = {NULL, NULL}; int allocated_holidays = 1; @@ -1065,17 +1065,17 @@ array_busday_count(PyObject *NPY_UNUSED(self), &dates_end_in, &PyArray_WeekMaskConverter, &weekmask[0], &PyArray_HolidaysConverter, &holidays, - &NpyBusinessDayDef_Type, &busdaydef, + &NpyBusDayCalendar_Type, &busdaycal, &out_in)) { goto fail; } - /* Make sure only one of the weekmask/holidays and busdaydef is supplied */ - if (busdaydef != NULL) { + /* Make sure only one of the weekmask/holidays and busdaycal is supplied */ + if (busdaycal != NULL) { if (weekmask[0] != 2 || holidays.begin != NULL) { PyErr_SetString(PyExc_ValueError, "Cannot supply both the weekmask/holidays and the " - "busdaydef parameters to busday_count()"); + "busdaycal parameters to busday_count()"); goto fail; } @@ -1083,9 +1083,9 @@ array_busday_count(PyObject *NPY_UNUSED(self), allocated_holidays = 0; /* Copy the private normalized weekmask/holidays data */ - holidays = busdaydef->holidays; - busdays_in_weekmask = busdaydef->busdays_in_weekmask; - memcpy(weekmask, busdaydef->weekmask, 7); + holidays = busdaycal->holidays; + busdays_in_weekmask = busdaycal->busdays_in_weekmask; + memcpy(weekmask, busdaycal->weekmask, 7); } else { /* @@ -1192,13 +1192,13 @@ array_is_busday(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) { char *kwlist[] = {"dates", - "weekmask", "holidays", "busdaydef", "out", NULL}; + "weekmask", "holidays", "busdaycal", "out", NULL}; PyObject *dates_in = NULL, *out_in = NULL; PyArrayObject *dates = NULL,*out = NULL, *ret; npy_bool weekmask[7] = {2, 1, 1, 1, 1, 0, 0}; - NpyBusinessDayDef *busdaydef = NULL; + NpyBusDayCalendar *busdaycal = NULL; int i, busdays_in_weekmask; npy_holidayslist holidays = {NULL, NULL}; int allocated_holidays = 1; @@ -1208,17 +1208,17 @@ array_is_busday(PyObject *NPY_UNUSED(self), &dates_in, &PyArray_WeekMaskConverter, &weekmask[0], &PyArray_HolidaysConverter, &holidays, - &NpyBusinessDayDef_Type, &busdaydef, + &NpyBusDayCalendar_Type, &busdaycal, &out_in)) { goto fail; } - /* Make sure only one of the weekmask/holidays and busdaydef is supplied */ - if (busdaydef != NULL) { + /* Make sure only one of the weekmask/holidays and busdaycal is supplied */ + if (busdaycal != NULL) { if (weekmask[0] != 2 || holidays.begin != NULL) { PyErr_SetString(PyExc_ValueError, "Cannot supply both the weekmask/holidays and the " - "busdaydef parameters to is_busday()"); + "busdaycal parameters to is_busday()"); goto fail; } @@ -1226,9 +1226,9 @@ array_is_busday(PyObject *NPY_UNUSED(self), allocated_holidays = 0; /* Copy the private normalized weekmask/holidays data */ - holidays = busdaydef->holidays; - busdays_in_weekmask = busdaydef->busdays_in_weekmask; - memcpy(weekmask, busdaydef->weekmask, 7); + holidays = busdaycal->holidays; + busdays_in_weekmask = busdaycal->busdays_in_weekmask; + memcpy(weekmask, busdaycal->weekmask, 7); } else { /* diff --git a/numpy/core/src/multiarray/datetime_busdaydef.c b/numpy/core/src/multiarray/datetime_busdaycal.c similarity index 93% rename from numpy/core/src/multiarray/datetime_busdaydef.c rename to numpy/core/src/multiarray/datetime_busdaycal.c index 396bf82bec9e..3d5bbe1ef1ea 100644 --- a/numpy/core/src/multiarray/datetime_busdaydef.c +++ b/numpy/core/src/multiarray/datetime_busdaycal.c @@ -1,6 +1,6 @@ /* * This file implements an object encapsulating a business day - * definition for accelerating NumPy datetime business day functions. + * calendar object for accelerating NumPy datetime business day functions. * * Written by Mark Wiebe (mwwiebe@gmail.com) * Copyright (c) 2011 by Enthought, Inc. @@ -21,7 +21,7 @@ #include "lowlevel_strided_loops.h" #include "_datetime.h" #include "datetime_busday.h" -#include "datetime_busdaydef.h" +#include "datetime_busdaycal.h" NPY_NO_EXPORT int PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask) @@ -332,12 +332,12 @@ PyArray_HolidaysConverter(PyObject *dates_in, npy_holidayslist *holidays) } static PyObject * -busdaydef_new(PyTypeObject *subtype, +busdaycalendar_new(PyTypeObject *subtype, PyObject *NPY_UNUSED(args), PyObject *NPY_UNUSED(kwds)) { - NpyBusinessDayDef *self; + NpyBusDayCalendar *self; - self = (NpyBusinessDayDef *)subtype->tp_alloc(subtype, 0); + self = (NpyBusDayCalendar *)subtype->tp_alloc(subtype, 0); if (self != NULL) { /* Start with an empty holidays list */ self->holidays.begin = NULL; @@ -358,7 +358,7 @@ busdaydef_new(PyTypeObject *subtype, } static int -busdaydef_init(NpyBusinessDayDef *self, PyObject *args, PyObject *kwds) +busdaycalendar_init(NpyBusDayCalendar *self, PyObject *args, PyObject *kwds) { static char *kwlist[] = {"weekmask", "holidays", NULL}; int i, busdays_in_weekmask; @@ -382,7 +382,7 @@ busdaydef_init(NpyBusinessDayDef *self, PyObject *args, PyObject *kwds) /* Parse the parameters */ if (!PyArg_ParseTupleAndKeywords(args, kwds, - "|O&O&:busdaydef", kwlist, + "|O&O&:busdaycal", kwlist, &PyArray_WeekMaskConverter, &self->weekmask[0], &PyArray_HolidaysConverter, &self->holidays)) { return -1; @@ -400,7 +400,7 @@ busdaydef_init(NpyBusinessDayDef *self, PyObject *args, PyObject *kwds) if (self->busdays_in_weekmask == 0) { PyErr_SetString(PyExc_ValueError, - "Cannot construct a numpy.busdaydef with a weekmask of " + "Cannot construct a numpy.busdaycal with a weekmask of " "all zeros"); return -1; } @@ -409,7 +409,7 @@ busdaydef_init(NpyBusinessDayDef *self, PyObject *args, PyObject *kwds) } static void -busdaydef_dealloc(NpyBusinessDayDef *self) +busdaycalendar_dealloc(NpyBusDayCalendar *self) { /* Clear the holidays */ if (self->holidays.begin != NULL) { @@ -422,7 +422,7 @@ busdaydef_dealloc(NpyBusinessDayDef *self) } static PyObject * -busdaydef_weekmask_get(NpyBusinessDayDef *self) +busdaycalendar_weekmask_get(NpyBusDayCalendar *self) { PyArrayObject *ret; npy_intp size = 7; @@ -440,7 +440,7 @@ busdaydef_weekmask_get(NpyBusinessDayDef *self) } static PyObject * -busdaydef_holidays_get(NpyBusinessDayDef *self) +busdaycalendar_holidays_get(NpyBusDayCalendar *self) { PyArrayObject *ret; PyArray_Descr *date_dtype; @@ -467,29 +467,29 @@ busdaydef_holidays_get(NpyBusinessDayDef *self) return (PyObject *)ret; } -static PyGetSetDef busdaydef_getsets[] = { +static PyGetSetDef busdaycalendar_getsets[] = { {"weekmask", - (getter)busdaydef_weekmask_get, + (getter)busdaycalendar_weekmask_get, NULL, NULL, NULL}, {"holidays", - (getter)busdaydef_holidays_get, + (getter)busdaycalendar_holidays_get, NULL, NULL, NULL}, {NULL, NULL, NULL, NULL, NULL} }; -NPY_NO_EXPORT PyTypeObject NpyBusinessDayDef_Type = { +NPY_NO_EXPORT PyTypeObject NpyBusDayCalendar_Type = { #if defined(NPY_PY3K) PyVarObject_HEAD_INIT(NULL, 0) #else PyObject_HEAD_INIT(NULL) 0, /* ob_size */ #endif - "numpy.busdaydef", /* tp_name */ - sizeof(NpyBusinessDayDef), /* tp_basicsize */ + "numpy.busdaycalendar", /* tp_name */ + sizeof(NpyBusDayCalendar), /* tp_basicsize */ 0, /* tp_itemsize */ /* methods */ - (destructor)busdaydef_dealloc, /* tp_dealloc */ + (destructor)busdaycalendar_dealloc, /* tp_dealloc */ 0, /* tp_print */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -518,15 +518,15 @@ NPY_NO_EXPORT PyTypeObject NpyBusinessDayDef_Type = { 0, /* tp_iternext */ 0, /* tp_methods */ 0, /* tp_members */ - busdaydef_getsets, /* tp_getset */ + busdaycalendar_getsets, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ - (initproc)busdaydef_init, /* tp_init */ + (initproc)busdaycalendar_init, /* tp_init */ 0, /* tp_alloc */ - busdaydef_new, /* tp_new */ + busdaycalendar_new, /* tp_new */ 0, /* tp_free */ 0, /* tp_is_gc */ 0, /* tp_bases */ diff --git a/numpy/core/src/multiarray/datetime_busdaydef.h b/numpy/core/src/multiarray/datetime_busdaycal.h similarity index 90% rename from numpy/core/src/multiarray/datetime_busdaydef.h rename to numpy/core/src/multiarray/datetime_busdaycal.h index b5f100910e91..8401f38b6fe8 100644 --- a/numpy/core/src/multiarray/datetime_busdaydef.h +++ b/numpy/core/src/multiarray/datetime_busdaycal.h @@ -24,12 +24,9 @@ typedef struct { npy_holidayslist holidays; int busdays_in_weekmask; npy_bool weekmask[7]; -} NpyBusinessDayDef; +} NpyBusDayCalendar; -NPY_NO_EXPORT PyTypeObject NpyBusinessDayDef_Type; - -#define NpyBusinessDayDef_Check(op) PyObject_TypeCheck(op, \ - &NpyBusinessDayDef_Type) +NPY_NO_EXPORT PyTypeObject NpyBusDayCalendar_Type; /* * Converts a Python input into a 7-element weekmask, where 0 means diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 60ee9dbe8032..3b136e072c00 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -46,7 +46,7 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0; #include "nditer_pywrap.h" #include "_datetime.h" #include "datetime_busday.h" -#include "datetime_busdaydef.h" +#include "datetime_busdaycal.h" /* Only here for API compatibility */ NPY_NO_EXPORT PyTypeObject PyBigArray_Type; @@ -3922,8 +3922,8 @@ PyMODINIT_FUNC initmultiarray(void) { if (PyType_Ready(&PyArrayFlags_Type) < 0) { return RETVAL; } - NpyBusinessDayDef_Type.tp_new = PyType_GenericNew; - if (PyType_Ready(&NpyBusinessDayDef_Type) < 0) { + NpyBusDayCalendar_Type.tp_new = PyType_GenericNew; + if (PyType_Ready(&NpyBusDayCalendar_Type) < 0) { return RETVAL; } /* FIXME @@ -4005,9 +4005,10 @@ PyMODINIT_FUNC initmultiarray(void) { Py_INCREF(&PyArrayFlags_Type); PyDict_SetItemString(d, "flagsobj", (PyObject *)&PyArrayFlags_Type); - /* Business day definition object */ - Py_INCREF(&NpyBusinessDayDef_Type); - PyDict_SetItemString(d, "busdaydef", (PyObject *)&NpyBusinessDayDef_Type); + /* Business day calendar object */ + Py_INCREF(&NpyBusDayCalendar_Type); + PyDict_SetItemString(d, "busdaycalendar", + (PyObject *)&NpyBusDayCalendar_Type); set_flaginfo(d); diff --git a/numpy/core/src/multiarray/multiarraymodule_onefile.c b/numpy/core/src/multiarray/multiarraymodule_onefile.c index 5f7cf17f3e73..4459e6b4ccc2 100644 --- a/numpy/core/src/multiarray/multiarraymodule_onefile.c +++ b/numpy/core/src/multiarray/multiarraymodule_onefile.c @@ -12,7 +12,7 @@ #include "datetime.c" #include "datetime_busday.c" -#include "datetime_busdaydef.c" +#include "datetime_busdaycal.c" #include "arraytypes.c" #include "hashdescr.c" diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index a3adcb2a6af6..599ece70d12e 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -1206,10 +1206,10 @@ def test_datetime_busday_offset(self): assert_equal(np.busday_offset('2007-04-07', -11, weekmask='SatSun'), np.datetime64('2007-02-25')) - def test_datetime_busdaydef(self): + def test_datetime_busdaycalendar(self): # Check that it removes NaT, duplicates, and weekends # and sorts the result. - bdd = np.busdaydef( + bdd = np.busdaycalendar( holidays=['NaT', '2011-01-17', '2011-03-06', 'NaT', '2011-12-26','2011-05-30','2011-01-17']) assert_equal(bdd.holidays, @@ -1218,7 +1218,7 @@ def test_datetime_busdaydef(self): assert_equal(bdd.weekmask, np.array([1,1,1,1,1,0,0], dtype='?')) # All-zeros weekmask should raise - assert_raises(ValueError, np.busdaydef, weekmask=[0,0,0,0,0,0,0]) + assert_raises(ValueError, np.busdaycalendar, weekmask=[0,0,0,0,0,0,0]) def test_datetime_busday_holidays_offset(self): # With exactly one holiday @@ -1290,7 +1290,7 @@ def test_datetime_busday_holidays_offset(self): holidays=['2011-10-10', '2011-11-11', '2011-11-24', '2011-12-25', '2011-05-30', '2011-02-21', '2011-12-26', '2012-01-02'] - bdd = np.busdaydef(weekmask='1111100', holidays=holidays) + bdd = np.busdaycalendar(weekmask='1111100', holidays=holidays) assert_equal( np.busday_offset('2011-10-03', 4, holidays=holidays), np.busday_offset('2011-10-03', 4)) @@ -1322,7 +1322,7 @@ def test_datetime_busday_holidays_offset(self): np.busday_offset('2011-10-03', 61, holidays=holidays), np.busday_offset('2011-10-03', 61 + 5)) assert_equal( - np.busday_offset('2011-10-03', 61, busdaydef=bdd), + np.busday_offset('2011-10-03', 61, busdaycal=bdd), np.busday_offset('2011-10-03', 61 + 5)) # A bigger backward jump across more than one week/holiday assert_equal( @@ -1353,14 +1353,14 @@ def test_datetime_busday_holidays_offset(self): np.busday_offset('2012-01-03', -57, holidays=holidays), np.busday_offset('2012-01-03', -57 - 5)) assert_equal( - np.busday_offset('2012-01-03', -57, busdaydef=bdd), + np.busday_offset('2012-01-03', -57, busdaycal=bdd), np.busday_offset('2012-01-03', -57 - 5)) - # Can't supply both a weekmask/holidays and busdaydef + # Can't supply both a weekmask/holidays and busdaycal assert_raises(ValueError, np.busday_offset, '2012-01-03', -15, - weekmask='1111100', busdaydef=bdd) + weekmask='1111100', busdaycal=bdd) assert_raises(ValueError, np.busday_offset, '2012-01-03', -15, - holidays=holidays, busdaydef=bdd) + holidays=holidays, busdaycal=bdd) # Roll with the holidays assert_equal( @@ -1393,25 +1393,25 @@ def test_datetime_busday_holidays_count(self): '2011-12-25', '2011-05-30', '2011-02-21', '2011-01-17', '2011-12-26', '2012-01-02', '2011-02-21', '2011-05-30', '2011-07-01', '2011-07-04', '2011-09-05', '2011-10-10'] - bdd = np.busdaydef(weekmask='1111100', holidays=holidays) + bdd = np.busdaycalendar(weekmask='1111100', holidays=holidays) # Validate against busday_offset broadcast against # a range of offsets dates = np.busday_offset('2011-01-01', np.arange(366), - roll='forward', busdaydef=bdd) - assert_equal(np.busday_count('2011-01-01', dates, busdaydef=bdd), + roll='forward', busdaycal=bdd) + assert_equal(np.busday_count('2011-01-01', dates, busdaycal=bdd), np.arange(366)) dates = np.busday_offset('2011-12-31', -np.arange(366), - roll='forward', busdaydef=bdd) - assert_equal(np.busday_count(dates, '2011-12-31', busdaydef=bdd), + roll='forward', busdaycal=bdd) + assert_equal(np.busday_count(dates, '2011-12-31', busdaycal=bdd), np.arange(366)) - # Can't supply both a weekmask/holidays and busdaydef + # Can't supply both a weekmask/holidays and busdaycal assert_raises(ValueError, np.busday_offset, '2012-01-03', '2012-02-03', - weekmask='1111100', busdaydef=bdd) + weekmask='1111100', busdaycal=bdd) assert_raises(ValueError, np.busday_offset, '2012-01-03', '2012-02-03', - holidays=holidays, busdaydef=bdd) + holidays=holidays, busdaycal=bdd) # Number of Mondays in March 2011 assert_equal(np.busday_count('2011-03', '2011-04', weekmask='Mon'), 4) @@ -1422,7 +1422,7 @@ def test_datetime_is_busday(self): '2011-12-26', '2012-01-02', '2011-02-21', '2011-05-30', '2011-07-01', '2011-07-04', '2011-09-05', '2011-10-10', 'NaT'] - bdd = np.busdaydef(weekmask='1111100', holidays=holidays) + bdd = np.busdaycalendar(weekmask='1111100', holidays=holidays) # Weekend/weekday tests assert_equal(np.is_busday('2011-01-01'), False) @@ -1430,7 +1430,7 @@ def test_datetime_is_busday(self): assert_equal(np.is_busday('2011-01-03'), True) # All the holidays are not business days - assert_equal(np.is_busday(holidays, busdaydef=bdd), + assert_equal(np.is_busday(holidays, busdaycal=bdd), np.zeros(len(holidays), dtype='?')) class TestDateTimeData(TestCase): From 31056d4279310e95d6e96792cc3ca3ba18df71e2 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Wed, 15 Jun 2011 16:11:38 -0500 Subject: [PATCH 28/34] DOC: datetime-feedback: Applying Ralf's feedback for the parameter conventions --- numpy/add_newdocs.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/numpy/add_newdocs.py b/numpy/add_newdocs.py index f1be98bd2857..58594b15aebf 100644 --- a/numpy/add_newdocs.py +++ b/numpy/add_newdocs.py @@ -5950,7 +5950,7 @@ def luf(lamdaexpr, *args, **kwargs): Parameters ---------- - weekmask : string or array_like of bool + weekmask : str or array_like of bool, optional A seven-element array indicating which of Monday through Sunday may be valid business days. May be specified as a list or array, like [1,1,1,1,1,0,0], a length-seven string like '1111100', or a string @@ -5958,7 +5958,7 @@ def luf(lamdaexpr, *args, **kwargs): string representation is most useful when only one day of the week is important, like 'Mon' if you want to calculate the date of Easter. - holidays : array_like of datetime64[D] + holidays : array_like of datetime64[D], optional An array of dates which should be blacked out from being considered as business days. They may be specified in any order, and NaT (not-a-time) dates are ignored. Internally, this list is normalized @@ -6011,7 +6011,7 @@ def luf(lamdaexpr, *args, **kwargs): ---------- dates : array_like of datetime64[D] The array of dates to process. - weekmask : string or array_like of bool + weekmask : str or array_like of bool, optional A seven-element array indicating which of Monday through Sunday may be valid business days. May be specified as a list or array, like [1,1,1,1,1,0,0], a length-seven string like '1111100', or a string @@ -6019,16 +6019,16 @@ def luf(lamdaexpr, *args, **kwargs): string representation is most useful when only one day of the week is important, like 'Mon' if you want to calculate the date of Easter. - holidays : array_like of datetime64[D] + holidays : array_like of datetime64[D], optional An array of dates which should be blacked out from being considered as business days. They may be specified in any order, and NaT (not-a-time) dates are ignored. Internally, this list is normalized into a form suited for fast business day calculations. - busdaycal : busdaycalendar + busdaycal : busdaycalendar, optional A `busdaycalendar` object which specifies the business days. If this parameter is provided, neither weekmask nor holidays may be provided. - out : array of bool + out : array of bool, optional If provided, this array is filled with the result. Returns @@ -6063,9 +6063,9 @@ def luf(lamdaexpr, *args, **kwargs): ---------- dates : array_like of datetime64[D] The array of dates to process. - offsets : array_like of integer + offsets : array_like of int The array of offsets, which is broadcast with ``dates``. - roll : {'raise', 'nat', 'forward', 'following', 'backward', 'preceding', 'modifiedfollowing', 'modifiedpreceding'} + roll : {'raise', 'nat', 'forward', 'following', 'backward', 'preceding', 'modifiedfollowing', 'modifiedpreceding'}, optional How to treat dates that do not fall on a business day. The default is 'raise'. @@ -6081,7 +6081,7 @@ def luf(lamdaexpr, *args, **kwargs): * 'modifiedpreceding' means to take the first business day earlier in time unless it is across a Month boundary, in which case to take the first business day later in time. - weekmask : string or array_like of bool + weekmask : str or array_like of bool, optional A seven-element array indicating which of Monday through Sunday may be valid business days. May be specified as a list or array, like [1,1,1,1,1,0,0], a length-seven string like '1111100', or a string @@ -6089,16 +6089,16 @@ def luf(lamdaexpr, *args, **kwargs): string representation is most useful when only one day of the week is important, like 'Mon' if you want to calculate the date of Easter. - holidays : array_like of datetime64[D] + holidays : array_like of datetime64[D], optional An array of dates which should be blacked out from being considered as business days. They may be specified in any order, and NaT (not-a-time) dates are ignored. Internally, this list is normalized into a form suited for fast business day calculations. - busdaycal : busdaycalendar + busdaycal : busdaycalendar, optional A `busdaycalendar` object which specifies the business days. If this parameter is provided, neither weekmask nor holidays may be provided. - out : array of datetime64[D] + out : array of datetime64[D], optional If provided, this array is filled with the result. Returns @@ -6154,7 +6154,7 @@ def luf(lamdaexpr, *args, **kwargs): enddates : array_like of datetime64[D] The array of the end dates for counting, which are excluded from the count themselves. - weekmask : string or array_like of bool + weekmask : str or array_like of bool, optional A seven-element array indicating which of Monday through Sunday may be valid business days. May be specified as a list or array, like [1,1,1,1,1,0,0], a length-seven string like '1111100', or a string @@ -6162,21 +6162,21 @@ def luf(lamdaexpr, *args, **kwargs): string representation is most useful when only one day of the week is important, like 'Mon' if you want to calculate the date of Easter. - holidays : array_like of datetime64[D] + holidays : array_like of datetime64[D], optional An array of dates which should be blacked out from being considered as business days. They may be specified in any order, and NaT (not-a-time) dates are ignored. Internally, this list is normalized into a form suited for fast business day calculations. - busdaycal : busdaycalendar + busdaycal : busdaycalendar, optional A `busdaycalendar` object which specifies the business days. If this parameter is provided, neither weekmask nor holidays may be provided. - out : array of int64 + out : array of int, optional If provided, this array is filled with the result. Returns ------- - out : array of int64 + out : array of int An array with a shape from broadcasting ``begindates`` and ``enddates`` together, containing the number of business days between the begin and end dates. From abfec953e9adfc5a74ca1761da3343cc643eba50 Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Thu, 16 Jun 2011 10:08:33 -0500 Subject: [PATCH 29/34] STY: datetime-feedback: Expand some comments and adjust spacing based on Chuck's feedback --- numpy/core/src/multiarray/datetime_busday.c | 18 +++++++++++++----- numpy/core/src/multiarray/datetime_busdaycal.c | 1 - 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/numpy/core/src/multiarray/datetime_busday.c b/numpy/core/src/multiarray/datetime_busday.c index 034e7f6abc00..2d83786a6b4b 100644 --- a/numpy/core/src/multiarray/datetime_busday.c +++ b/numpy/core/src/multiarray/datetime_busday.c @@ -41,7 +41,9 @@ get_day_of_week(npy_datetime date) * Returns 1 if the date is a holiday (contained in the sorted * list of dates), 0 otherwise. * - * The holidays list should be normalized. + * The holidays list should be normalized, which means any NaT (not-a-time) + * values, duplicates, and dates already excluded by the weekmask should + * be removed, and the list should be sorted. */ static int is_holiday(npy_datetime date, @@ -79,7 +81,9 @@ is_holiday(npy_datetime date, * holidays_begin = find_holiday_earliest_on_or_after(date, * holidays_begin, holidays_end); * - * The holidays list should be normalized. + * The holidays list should be normalized, which means any NaT (not-a-time) + * values, duplicates, and dates already excluded by the weekmask should + * be removed, and the list should be sorted. */ static npy_datetime * find_earliest_holiday_on_or_after(npy_datetime date, @@ -116,7 +120,9 @@ find_earliest_holiday_on_or_after(npy_datetime date, * holidays_end = find_holiday_earliest_after(date, * holidays_begin, holidays_end); * - * The holidays list should be normalized. + * The holidays list should be normalized, which means any NaT (not-a-time) + * values, duplicates, and dates already excluded by the weekmask should + * be removed, and the list should be sorted. */ static npy_datetime * find_earliest_holiday_after(npy_datetime date, @@ -283,7 +289,7 @@ apply_business_day_offset(npy_datetime date, npy_int64 offset, if (date == NPY_DATETIME_NAT) { return 0; } - + /* Now we're on a valid business day */ if (offset > 0) { /* Remove any earlier holidays */ @@ -782,7 +788,6 @@ is_business_day(PyArrayObject *dates, PyArrayObject *out, !is_holiday(date, holidays_begin, holidays_end) && date != NPY_DATETIME_NAT; - data_dates += stride_dates; data_out += stride_out; @@ -1029,6 +1034,7 @@ array_busday_offset(PyObject *NPY_UNUSED(self), } return out == NULL ? PyArray_Return(ret) : (PyObject *)ret; + fail: Py_XDECREF(dates); Py_XDECREF(offsets); @@ -1173,6 +1179,7 @@ array_busday_count(PyObject *NPY_UNUSED(self), } return out == NULL ? PyArray_Return(ret) : (PyObject *)ret; + fail: Py_XDECREF(dates_begin); Py_XDECREF(dates_end); @@ -1292,6 +1299,7 @@ array_is_busday(PyObject *NPY_UNUSED(self), } return out == NULL ? PyArray_Return(ret) : (PyObject *)ret; + fail: Py_XDECREF(dates); if (allocated_holidays && holidays.begin != NULL) { diff --git a/numpy/core/src/multiarray/datetime_busdaycal.c b/numpy/core/src/multiarray/datetime_busdaycal.c index 3d5bbe1ef1ea..5ad4ee5536cf 100644 --- a/numpy/core/src/multiarray/datetime_busdaycal.c +++ b/numpy/core/src/multiarray/datetime_busdaycal.c @@ -185,7 +185,6 @@ PyArray_WeekMaskConverter(PyObject *weekmask_in, npy_bool *weekmask) Py_DECREF(obj); return 0; - finish: Py_DECREF(obj); return 1; From e6bffff78cd34d4ba1963c28fff3e8537f6a857a Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Thu, 16 Jun 2011 12:02:29 -0500 Subject: [PATCH 30/34] BUG: dtype: Cleanups and fix parsing datetime dtypes with an endian specifier One fairly major improvement is that parsing with kind and a size now makes sure there isn't any garbage after the size. --- numpy/core/include/numpy/ufuncobject.h | 3 +- numpy/core/src/multiarray/arraytypes.c.src | 46 +++--- numpy/core/src/multiarray/datetime.c | 3 +- numpy/core/src/multiarray/descriptor.c | 179 ++++++++++++--------- numpy/core/tests/test_datetime.py | 27 +++- 5 files changed, 156 insertions(+), 102 deletions(-) diff --git a/numpy/core/include/numpy/ufuncobject.h b/numpy/core/include/numpy/ufuncobject.h index ae8f06827e6b..b4354b58cdc7 100644 --- a/numpy/core/include/numpy/ufuncobject.h +++ b/numpy/core/include/numpy/ufuncobject.h @@ -25,7 +25,8 @@ struct _tagPyUFuncObject; * type_tup: Either NULL, or the type_tup passed to the ufunc. * out_dtypes: An array which should be populated with new * references to (ufunc->nin + ufunc->nout) new - * dtypes, one for each input and output. + * dtypes, one for each input and output. These + * dtypes should all be in native-endian format. * out_innerloop: Should be populated with the correct ufunc inner * loop for the given type. * out_innerloopdata: Should be populated with the void* data to diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index f5f173d75500..e4dab1f7e6b7 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -381,7 +381,7 @@ UNICODE_setitem(PyObject *op, char *ov, PyArrayObject *ap) memcpy(ov, ptr, MIN(ap->descr->elsize, datalen)); #else if (!PyArray_ISALIGNED(ap)) { - buffer = _pya_malloc(ap->descr->elsize); + buffer = PyArray_malloc(ap->descr->elsize); if (buffer == NULL) { Py_DECREF(temp); PyErr_NoMemory(); @@ -396,7 +396,7 @@ UNICODE_setitem(PyObject *op, char *ov, PyArrayObject *ap) datalen <<= 2; if (!PyArray_ISALIGNED(ap)) { memcpy(ov, buffer, datalen); - _pya_free(buffer); + PyArray_free(buffer); } #endif /* Fill in the rest of the space with 0 */ @@ -2110,7 +2110,7 @@ UNICODE_nonzero (PyArray_UCS4 *ip, PyArrayObject *ap) char *buffer = NULL; if ((!PyArray_ISNOTSWAPPED(ap)) || (!PyArray_ISALIGNED(ap))) { - buffer = _pya_malloc(ap->descr->elsize); + buffer = PyArray_malloc(ap->descr->elsize); if (buffer == NULL) { return nonz; } @@ -2128,7 +2128,7 @@ UNICODE_nonzero (PyArray_UCS4 *ip, PyArrayObject *ap) } ip++; } - _pya_free(buffer); + PyArray_free(buffer); return nonz; } @@ -2486,7 +2486,7 @@ VOID_compare(char *ip1, char *ip2, PyArrayObject *ap) if ((swap) || (new->alignment > 1)) { if ((swap) || (((intp)(nip1) % new->alignment) != 0)) { /* create buffer and copy */ - nip1 = _pya_malloc(new->elsize); + nip1 = PyArray_malloc(new->elsize); if (nip1 == NULL) { goto finish; } @@ -2496,10 +2496,10 @@ VOID_compare(char *ip1, char *ip2, PyArrayObject *ap) } if ((swap) || (((intp)(nip2) % new->alignment) != 0)) { /* copy data to a buffer */ - nip2 = _pya_malloc(new->elsize); + nip2 = PyArray_malloc(new->elsize); if (nip2 == NULL) { if (nip1 != ip1+offset) { - _pya_free(nip1); + PyArray_free(nip1); } goto finish; } @@ -2511,10 +2511,10 @@ VOID_compare(char *ip1, char *ip2, PyArrayObject *ap) res = new->f->compare(nip1, nip2, ap); if ((swap) || (new->alignment > 1)) { if (nip1 != ip1+offset) { - _pya_free(nip1); + PyArray_free(nip1); } if (nip2 != ip2+offset) { - _pya_free(nip2); + PyArray_free(nip2); } } if (res != 0) { @@ -2647,7 +2647,7 @@ static int { intp i; int elsize = aip->descr->elsize; - @type@ *mp = (@type@ *)_pya_malloc(elsize); + @type@ *mp = (@type@ *)PyArray_malloc(elsize); if (mp==NULL) return 0; memcpy(mp, ip, elsize); @@ -2659,7 +2659,7 @@ static int *max_ind=i; } } - _pya_free(mp); + PyArray_free(mp); return 0; } @@ -3433,7 +3433,7 @@ _init_datetime_descr(PyArray_Descr *descr) PyArray_DatetimeMetaData *dt_data; PyObject *cobj; - dt_data = _pya_malloc(sizeof(PyArray_DatetimeMetaData)); + dt_data = PyArray_malloc(sizeof(PyArray_DatetimeMetaData)); dt_data->base = NPY_DATETIME_DEFAULTUNIT; dt_data->num = 1; dt_data->events = 1; @@ -3487,35 +3487,35 @@ PyArray_DescrFromType(int type) { PyArray_Descr *ret = NULL; - if (type < PyArray_NTYPES) { + if (type < NPY_NTYPES) { ret = _builtin_descrs[type]; } - else if (type == PyArray_NOTYPE) { + else if (type == NPY_NOTYPE) { /* * This needs to not raise an error so - * that PyArray_DescrFromType(PyArray_NOTYPE) + * that PyArray_DescrFromType(NPY_NOTYPE) * works for backwards-compatible C-API */ return NULL; } - else if ((type == PyArray_CHAR) || (type == PyArray_CHARLTR)) { - ret = PyArray_DescrNew(_builtin_descrs[PyArray_STRING]); + else if ((type == NPY_CHAR) || (type == NPY_CHARLTR)) { + ret = PyArray_DescrNew(_builtin_descrs[NPY_STRING]); if (ret == NULL) { return NULL; } ret->elsize = 1; - ret->type = PyArray_CHARLTR; + ret->type = NPY_CHARLTR; return ret; } else if (PyTypeNum_ISUSERDEF(type)) { - ret = userdescrs[type - PyArray_USERDEF]; + ret = userdescrs[type - NPY_USERDEF]; } else { - int num = PyArray_NTYPES; + int num = NPY_NTYPES; if (type < _MAX_LETTER) { num = (int) _letter_to_num[type]; } - if (num >= PyArray_NTYPES) { + if (num >= NPY_NTYPES) { ret = NULL; } else { @@ -3606,9 +3606,9 @@ set_typeinfo(PyObject *dict) * CFLOAT, CDOUBLE, CLONGDOUBLE, OBJECT, STRING, UNICODE, VOID, * DATETIME,TIMEDELTA# */ - _letter_to_num[PyArray_@name@LTR] = PyArray_@name@; + _letter_to_num[NPY_@name@LTR] = NPY_@name@; /**end repeat**/ - _letter_to_num[PyArray_STRINGLTR2] = PyArray_STRING; + _letter_to_num[NPY_STRINGLTR2] = NPY_STRING; /**begin repeat * #name = BOOL, BYTE, UBYTE, SHORT, USHORT, INT, UINT, diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 78a5032f5d62..16ac5c4fbf38 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -138,7 +138,8 @@ days_to_yearsdays(npy_int64 *days_) { const npy_int64 days_per_400years = (400*365 + 100 - 4 + 1); /* Adjust so it's relative to the year 2000 (divisible by 400) */ - npy_int64 days = (*days_) - (365*30 + 7), year; + npy_int64 days = (*days_) - (365*30 + 7); + npy_int64 year; /* Break down the 400 year cycle to get the year and day within the year */ if (days >= 0) { diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 1fe8f748da34..6016917115c4 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -930,24 +930,27 @@ PyArray_DescrConverter2(PyObject *obj, PyArray_Descr **at) * * This is the central code that converts Python objects to * Type-descriptor objects that are used throughout numpy. - * new reference in *at + * + * Returns a new reference in *at, but the returned should not be + * modified as it may be one of the canonical immutable objects or + * a reference to the input obj. */ NPY_NO_EXPORT int PyArray_DescrConverter(PyObject *obj, PyArray_Descr **at) { - char *type; int check_num = PyArray_NOTYPE + 10; - int len; PyObject *item; int elsize = 0; char endian = '='; *at = NULL; + /* default */ if (obj == Py_None) { *at = PyArray_DescrFromType(PyArray_DEFAULT); return PY_SUCCEED; } + if (PyArray_DescrCheck(obj)) { *at = (PyArray_Descr *)obj; Py_INCREF(*at); @@ -957,50 +960,43 @@ PyArray_DescrConverter(PyObject *obj, PyArray_Descr **at) if (PyType_Check(obj)) { if (PyType_IsSubtype((PyTypeObject *)obj, &PyGenericArrType_Type)) { *at = PyArray_DescrFromTypeObject(obj); - if (*at) { - return PY_SUCCEED; - } - else { - return PY_FAIL; - } + return (*at) ? PY_SUCCEED : PY_FAIL; } - check_num = PyArray_OBJECT; + check_num = NPY_OBJECT; #if !defined(NPY_PY3K) if (obj == (PyObject *)(&PyInt_Type)) { - check_num = PyArray_LONG; + check_num = NPY_LONG; } else if (obj == (PyObject *)(&PyLong_Type)) { - check_num = PyArray_LONGLONG; + check_num = NPY_LONGLONG; } #else if (obj == (PyObject *)(&PyLong_Type)) { - check_num = PyArray_LONG; + check_num = NPY_LONG; } #endif else if (obj == (PyObject *)(&PyFloat_Type)) { - check_num = PyArray_DOUBLE; + check_num = NPY_DOUBLE; } else if (obj == (PyObject *)(&PyComplex_Type)) { - check_num = PyArray_CDOUBLE; + check_num = NPY_CDOUBLE; } else if (obj == (PyObject *)(&PyBool_Type)) { - check_num = PyArray_BOOL; + check_num = NPY_BOOL; } else if (obj == (PyObject *)(&PyBytes_Type)) { - check_num = PyArray_STRING; + check_num = NPY_STRING; } else if (obj == (PyObject *)(&PyUnicode_Type)) { - check_num = PyArray_UNICODE; + check_num = NPY_UNICODE; } #if defined(NPY_PY3K) else if (obj == (PyObject *)(&PyMemoryView_Type)) { - check_num = PyArray_VOID; - } #else else if (obj == (PyObject *)(&PyBuffer_Type)) { - check_num = PyArray_VOID; - } #endif + check_num = NPY_VOID; + } else { *at = _arraydescr_fromobj(obj); if (*at) { @@ -1026,61 +1022,102 @@ PyArray_DescrConverter(PyObject *obj, PyArray_Descr **at) } if (PyBytes_Check(obj)) { + char *type = NULL; + Py_ssize_t len = 0; + /* Check for a string typecode. */ - type = PyBytes_AS_STRING(obj); - len = PyBytes_GET_SIZE(obj); - if (len <= 0) { - goto fail; + if (PyBytes_AsStringAndSize(obj, &type, &len) < 0) { + goto error; } - /* check for datetime format */ - if (is_datetime_typestr(type, len)) { - *at = parse_dtype_from_datetime_typestr(type, len); - return (*at) ? PY_SUCCEED : PY_FAIL; + + /* Empty string is invalid */ + if (len == 0) { + goto fail; } + /* check for commas present or first (or second) element a digit */ if (_check_for_commastring(type, len)) { *at = _convert_from_commastring(obj, 0); return (*at) ? PY_SUCCEED : PY_FAIL; } - check_num = (int) type[0]; - if ((char) check_num == '>' - || (char) check_num == '<' - || (char) check_num == '|' - || (char) check_num == '=') { - if (len <= 1) { - goto fail; - } - endian = (char) check_num; - type++; len--; - check_num = (int) type[0]; - if (endian == '|') { + + /* Process the endian character */ + switch (type[0]) { + case '>': + case '<': + case '=': + endian = type[0]; + ++type; + --len; + break; + + case '|': endian = '='; - } + ++type; + --len; + break; } - if (len > 1) { - elsize = atoi(type + 1); - if (elsize == 0) { - check_num = PyArray_NOTYPE+10; - } - /* - * When specifying length of UNICODE - * the number of characters is given to match - * the STRING interface. Each character can be - * more than one byte and itemsize must be - * the number of bytes. - */ - else if (check_num == PyArray_UNICODELTR) { - elsize <<= 2; - } - /* Support for generic processing c4, i4, f8, etc...*/ - else if ((check_num != PyArray_STRINGLTR) - && (check_num != PyArray_VOIDLTR) - && (check_num != PyArray_STRINGLTR2)) { - check_num = PyArray_TypestrConvert(elsize, check_num); - if (check_num == PyArray_NOTYPE) { - check_num += 10; + + /* Just an endian character is invalid */ + if (len == 0) { + goto fail; + } + + /* Check for datetime format */ + if (is_datetime_typestr(type, len)) { + *at = parse_dtype_from_datetime_typestr(type, len); + return (*at) ? PY_SUCCEED : PY_FAIL; + } + + /* A typecode like 'd' */ + if (len == 1) { + check_num = type[0]; + } + /* A kind + size like 'f8' */ + else { + char *typeend = NULL; + int kind; + + /* Parse the integer, make sure it's the rest of the string */ + elsize = (int)strtol(type + 1, &typeend, 10); + if (typeend - type == len) { + + kind = type[0]; + switch (kind) { + case NPY_STRINGLTR: + case NPY_STRINGLTR2: + check_num = NPY_STRING; + break; + + /* + * When specifying length of UNICODE + * the number of characters is given to match + * the STRING interface. Each character can be + * more than one byte and itemsize must be + * the number of bytes. + */ + case NPY_UNICODELTR: + check_num = NPY_UNICODE; + elsize <<= 2; + break; + + case NPY_VOIDLTR: + check_num = NPY_VOID; + break; + + default: + if (elsize == 0) { + check_num = NPY_NOTYPE+10; + } + /* Support for generic processing c8, i4, f8, etc...*/ + else { + check_num = PyArray_TypestrConvert(elsize, kind); + if (check_num == NPY_NOTYPE) { + check_num += 10; + } + elsize = 0; + } } - elsize = 0; } } } @@ -1133,12 +1170,8 @@ PyArray_DescrConverter(PyObject *obj, PyArray_Descr **at) if (PyErr_Occurred()) { goto fail; } - /* if (check_num == PyArray_NOTYPE) { - return PY_FAIL; - } - */ - finish: +finish: if ((check_num == PyArray_NOTYPE + 10) || (*at = PyArray_DescrFromType(check_num)) == NULL) { PyErr_Clear(); @@ -1176,7 +1209,7 @@ PyArray_DescrConverter(PyObject *obj, PyArray_Descr **at) } return PY_SUCCEED; - fail: +fail: if (PyBytes_Check(obj)) { PyErr_Format(PyExc_TypeError, "data type \"%s\" not understood", PyBytes_AS_STRING(obj)); @@ -1184,6 +1217,8 @@ PyArray_DescrConverter(PyObject *obj, PyArray_Descr **at) else { PyErr_SetString(PyExc_TypeError, "data type not understood"); } + +error: *at = NULL; return PY_FAIL; } diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index 599ece70d12e..d14b4bc07f28 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -18,6 +18,21 @@ def test_datetime_dtype_creation(self): # Generic units shouldn't add [] to the end assert_equal(str(np.dtype("M8")), "datetime64") + # Should be possible to specify the endianness + assert_equal(np.dtype("=M8"), np.dtype("M8")) + assert_equal(np.dtype("=M8[s]"), np.dtype("M8[s]")) + assert_(np.dtype(">M8") == np.dtype("M8") or + np.dtype("M8[D]") == np.dtype("M8[D]") or + np.dtype("m8") == np.dtype("m8") or + np.dtype("m8[D]") == np.dtype("m8[D]") or + np.dtype(" Date: Thu, 16 Jun 2011 12:27:35 -0500 Subject: [PATCH 31/34] BUG: core: promote_types wasn't always returning NBO data types --- numpy/core/src/multiarray/convert_datatype.c | 84 ++++++++------------ numpy/core/tests/test_numeric.py | 23 ++++++ 2 files changed, 57 insertions(+), 50 deletions(-) diff --git a/numpy/core/src/multiarray/convert_datatype.c b/numpy/core/src/multiarray/convert_datatype.c index 73d65c12d2c9..9e0b93a56fbf 100644 --- a/numpy/core/src/multiarray/convert_datatype.c +++ b/numpy/core/src/multiarray/convert_datatype.c @@ -565,6 +565,22 @@ promote_types(PyArray_Descr *type1, PyArray_Descr *type2, } +/* + * Returns a new reference to type if it is already NBO, otherwise + * returns a copy converted to NBO. + */ +static PyArray_Descr * +ensure_dtype_nbo(PyArray_Descr *type) +{ + if (PyArray_ISNBO(type->byteorder)) { + Py_INCREF(type); + return type; + } + else { + return PyArray_DescrNewByteorder(type, NPY_NATIVE); + } +} + /*NUMPY_API * Produces the smallest size and lowest kind type to which both * input types can be cast. @@ -594,23 +610,11 @@ PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2) if (PyArray_CanCastTo(type2, type1)) { /* Promoted types are always native byte order */ - if (PyArray_ISNBO(type1->byteorder)) { - Py_INCREF(type1); - return type1; - } - else { - return PyArray_DescrNewByteorder(type1, NPY_NATIVE); - } + return ensure_dtype_nbo(type1); } else if (PyArray_CanCastTo(type1, type2)) { /* Promoted types are always native byte order */ - if (PyArray_ISNBO(type2->byteorder)) { - Py_INCREF(type2); - return type2; - } - else { - return PyArray_DescrNewByteorder(type2, NPY_NATIVE); - } + return ensure_dtype_nbo(type2); } /* Convert the 'kind' char into a scalar kind */ @@ -691,8 +695,7 @@ PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2) /* BOOL can convert to anything except datetime/void */ case NPY_BOOL: if (type_num2 != NPY_DATETIME && type_num2 != NPY_VOID) { - Py_INCREF(type2); - return type2; + return ensure_dtype_nbo(type2); } else { break; @@ -701,18 +704,15 @@ PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2) case NPY_STRING: if (type_num2 == NPY_STRING) { if (type1->elsize > type2->elsize) { - Py_INCREF(type1); - return type1; + return ensure_dtype_nbo(type1); } else { - Py_INCREF(type2); - return type2; + return ensure_dtype_nbo(type2); } } else if (type_num2 == NPY_UNICODE) { if (type2->elsize >= type1->elsize * 4) { - Py_INCREF(type2); - return type2; + return ensure_dtype_nbo(type2); } else { PyArray_Descr *d = PyArray_DescrNewFromType(NPY_UNICODE); @@ -725,24 +725,20 @@ PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2) } /* Allow NUMBER -> STRING */ else if (PyTypeNum_ISNUMBER(type_num2)) { - Py_INCREF(type1); - return type1; + return ensure_dtype_nbo(type1); } case NPY_UNICODE: if (type_num2 == NPY_UNICODE) { if (type1->elsize > type2->elsize) { - Py_INCREF(type1); - return type1; + return ensure_dtype_nbo(type1); } else { - Py_INCREF(type2); - return type2; + return ensure_dtype_nbo(type2); } } else if (type_num2 == NPY_STRING) { if (type1->elsize >= type2->elsize * 4) { - Py_INCREF(type1); - return type1; + return ensure_dtype_nbo(type1); } else { PyArray_Descr *d = PyArray_DescrNewFromType(NPY_UNICODE); @@ -755,8 +751,7 @@ PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2) } /* Allow NUMBER -> UNICODE */ else if (PyTypeNum_ISNUMBER(type_num2)) { - Py_INCREF(type1); - return type1; + return ensure_dtype_nbo(type1); } break; case NPY_DATETIME: @@ -772,8 +767,7 @@ PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2) case NPY_BOOL: if (type_num1 != NPY_DATETIME && type_num1 != NPY_TIMEDELTA && type_num1 != NPY_VOID) { - Py_INCREF(type1); - return type1; + return ensure_dtype_nbo(type1); } else { break; @@ -781,35 +775,25 @@ PyArray_PromoteTypes(PyArray_Descr *type1, PyArray_Descr *type2) case NPY_STRING: /* Allow NUMBER -> STRING */ if (PyTypeNum_ISNUMBER(type_num1)) { - Py_INCREF(type2); - return type2; + return ensure_dtype_nbo(type2); } case NPY_UNICODE: /* Allow NUMBER -> UNICODE */ if (PyTypeNum_ISNUMBER(type_num1)) { - Py_INCREF(type2); - return type2; - } - break; - case NPY_DATETIME: - if (PyTypeNum_ISINTEGER(type_num1)) { - Py_INCREF(type2); - return type2; + return ensure_dtype_nbo(type2); } break; case NPY_TIMEDELTA: if (PyTypeNum_ISINTEGER(type_num1) || PyTypeNum_ISFLOAT(type_num1)) { - Py_INCREF(type2); - return type2; + return ensure_dtype_nbo(type2); } break; } - /* For equivalent types we can return either */ - if (PyArray_EquivTypes(type1, type2)) { - Py_INCREF(type1); - return type1; + /* For types equivalent up to endianness, can return either */ + if (PyArray_CanCastTypeTo(type1, type2, NPY_EQUIV_CASTING)) { + return ensure_dtype_nbo(type1); } /* TODO: Also combine fields, subarrays, strings, etc */ diff --git a/numpy/core/tests/test_numeric.py b/numpy/core/tests/test_numeric.py index 945e05001200..75ecd398a15f 100644 --- a/numpy/core/tests/test_numeric.py +++ b/numpy/core/tests/test_numeric.py @@ -464,6 +464,29 @@ def res_type(a, b): def test_result_type(self): self.check_promotion_cases(np.result_type) + def test_promote_types_endian(self): + # promote_types should always return native-endian types + assert_equal(np.promote_types('i8', '>i8'), np.dtype('i8')) + + assert_equal(np.promote_types('>i8', '>U16'), np.dtype('U16')) + assert_equal(np.promote_types('U16', '>i8'), np.dtype('U16')) + assert_equal(np.promote_types('S5', '>U8'), np.dtype('U8')) + assert_equal(np.promote_types('U8', '>S5'), np.dtype('U8')) + assert_equal(np.promote_types('U8', '>U5'), np.dtype('U8')) + + assert_equal(np.promote_types('M8', '>M8'), np.dtype('M8')) + assert_equal(np.promote_types('m8', '>m8'), np.dtype('m8')) + + def test_can_cast(self): assert_(np.can_cast(np.int32, np.int64)) assert_(np.can_cast(np.float64, np.complex)) From afe25c18c0c42cc4281068e3c91d471176b7b11d Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Thu, 16 Jun 2011 12:42:32 -0500 Subject: [PATCH 32/34] BUG: ufunc: Type promotion output must always be in NBO (fixes #1867) --- numpy/core/src/umath/ufunc_object.c | 133 ++++++++++++++++++++-------- numpy/core/tests/test_ufunc.py | 13 +++ 2 files changed, 110 insertions(+), 36 deletions(-) diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index c166cb1b5b6c..2e46fc51aeda 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -1705,6 +1705,22 @@ find_specified_ufunc_inner_loop(PyUFuncObject *self, return -1; } +/* + * Returns a new reference to type if it is already NBO, otherwise + * returns a copy converted to NBO. + */ +static PyArray_Descr * +ensure_dtype_nbo(PyArray_Descr *type) +{ + if (PyArray_ISNBO(type->byteorder)) { + Py_INCREF(type); + return type; + } + else { + return PyArray_DescrNewByteorder(type, NPY_NATIVE); + } +} + /*UFUNC_API * * This function applies the default type resolution rules @@ -1827,9 +1843,12 @@ PyUFunc_SimpleBinaryComparisonTypeResolution(PyUFuncObject *ufunc, return -1; } - out_dtypes[0] = (PyArray_Descr *)PyTuple_GET_ITEM(type_tup, 0); + out_dtypes[0] = ensure_dtype_nbo( + (PyArray_Descr *)PyTuple_GET_ITEM(type_tup, 0)); + if (out_dtypes[0] == NULL) { + return -1; + } out_dtypes[1] = out_dtypes[0]; - Py_INCREF(out_dtypes[0]); Py_INCREF(out_dtypes[1]); } @@ -1924,8 +1943,10 @@ PyUFunc_SimpleUnaryOperationTypeResolution(PyUFuncObject *ufunc, if (type_tup == NULL) { /* Input types are the result type */ - out_dtypes[0] = PyArray_DESCR(operands[0]); - Py_INCREF(out_dtypes[0]); + out_dtypes[0] = ensure_dtype_nbo(PyArray_DESCR(operands[0])); + if (out_dtypes[0] == NULL) { + return -1; + } out_dtypes[1] = out_dtypes[0]; Py_INCREF(out_dtypes[1]); } @@ -1945,8 +1966,11 @@ PyUFunc_SimpleUnaryOperationTypeResolution(PyUFuncObject *ufunc, return -1; } - out_dtypes[0] = (PyArray_Descr *)PyTuple_GET_ITEM(type_tup, 0); - Py_INCREF(out_dtypes[0]); + out_dtypes[0] = ensure_dtype_nbo( + (PyArray_Descr *)PyTuple_GET_ITEM(type_tup, 0)); + if (out_dtypes[0] == NULL) { + return -1; + } out_dtypes[1] = out_dtypes[0]; Py_INCREF(out_dtypes[1]); } @@ -2060,8 +2084,11 @@ PyUFunc_SimpleBinaryOperationTypeResolution(PyUFuncObject *ufunc, return -1; } - out_dtypes[0] = (PyArray_Descr *)PyTuple_GET_ITEM(type_tup, 0); - Py_INCREF(out_dtypes[0]); + out_dtypes[0] = ensure_dtype_nbo( + (PyArray_Descr *)PyTuple_GET_ITEM(type_tup, 0)); + if (out_dtypes[0] == NULL) { + return -1; + } out_dtypes[1] = out_dtypes[0]; Py_INCREF(out_dtypes[1]); out_dtypes[2] = out_dtypes[0]; @@ -2233,8 +2260,6 @@ timedelta_dtype_with_copied_meta(PyArray_Descr *dtype) return ret; } - - /* * This function applies the type resolution rules for addition. * In particular, there are a number of special cases with datetime: @@ -2307,8 +2332,10 @@ PyUFunc_AdditionTypeResolution(PyUFuncObject *ufunc, /* m8[] + int => m8[] + m8[] */ else if (PyTypeNum_ISINTEGER(type_num2) || PyTypeNum_ISBOOL(type_num2)) { - out_dtypes[0] = PyArray_DESCR(operands[0]); - Py_INCREF(out_dtypes[0]); + out_dtypes[0] = ensure_dtype_nbo(PyArray_DESCR(operands[0])); + if (out_dtypes[0] == NULL) { + return -1; + } out_dtypes[1] = out_dtypes[0]; Py_INCREF(out_dtypes[1]); out_dtypes[2] = out_dtypes[0]; @@ -2341,14 +2368,18 @@ PyUFunc_AdditionTypeResolution(PyUFuncObject *ufunc, /* M8[] + int => M8[] + m8[] */ else if (PyTypeNum_ISINTEGER(type_num2) || PyTypeNum_ISBOOL(type_num2)) { + out_dtypes[0] = ensure_dtype_nbo(PyArray_DESCR(operands[0])); + if (out_dtypes[0] == NULL) { + return -1; + } /* Make a new NPY_TIMEDELTA, and copy type1's metadata */ out_dtypes[1] = timedelta_dtype_with_copied_meta( PyArray_DESCR(operands[0])); if (out_dtypes[1] == NULL) { + Py_DECREF(out_dtypes[0]); + out_dtypes[0] = NULL; return -1; } - out_dtypes[0] = PyArray_DESCR(operands[0]); - Py_INCREF(out_dtypes[0]); out_dtypes[2] = out_dtypes[0]; Py_INCREF(out_dtypes[2]); @@ -2361,8 +2392,10 @@ PyUFunc_AdditionTypeResolution(PyUFuncObject *ufunc, else if (PyTypeNum_ISINTEGER(type_num1) || PyTypeNum_ISBOOL(type_num1)) { /* int + m8[] => m8[] + m8[] */ if (type_num2 == NPY_TIMEDELTA) { - out_dtypes[0] = PyArray_DESCR(operands[1]); - Py_INCREF(out_dtypes[0]); + out_dtypes[0] = ensure_dtype_nbo(PyArray_DESCR(operands[1])); + if (out_dtypes[0] == NULL) { + return -1; + } out_dtypes[1] = out_dtypes[0]; Py_INCREF(out_dtypes[1]); out_dtypes[2] = out_dtypes[0]; @@ -2377,8 +2410,12 @@ PyUFunc_AdditionTypeResolution(PyUFuncObject *ufunc, if (out_dtypes[0] == NULL) { return -1; } - out_dtypes[1] = PyArray_DESCR(operands[1]); - Py_INCREF(out_dtypes[1]); + out_dtypes[1] = ensure_dtype_nbo(PyArray_DESCR(operands[1])); + if (out_dtypes[1] == NULL) { + Py_DECREF(out_dtypes[0]); + out_dtypes[0] = NULL; + return -1; + } out_dtypes[2] = out_dtypes[1]; Py_INCREF(out_dtypes[2]); @@ -2485,8 +2522,10 @@ PyUFunc_SubtractionTypeResolution(PyUFuncObject *ufunc, /* m8[] - int => m8[] - m8[] */ else if (PyTypeNum_ISINTEGER(type_num2) || PyTypeNum_ISBOOL(type_num2)) { - out_dtypes[0] = PyArray_DESCR(operands[0]); - Py_INCREF(out_dtypes[0]); + out_dtypes[0] = ensure_dtype_nbo(PyArray_DESCR(operands[0])); + if (out_dtypes[0] == NULL) { + return -1; + } out_dtypes[1] = out_dtypes[0]; Py_INCREF(out_dtypes[1]); out_dtypes[2] = out_dtypes[0]; @@ -2519,14 +2558,18 @@ PyUFunc_SubtractionTypeResolution(PyUFuncObject *ufunc, /* M8[] - int => M8[] - m8[] */ else if (PyTypeNum_ISINTEGER(type_num2) || PyTypeNum_ISBOOL(type_num2)) { + out_dtypes[0] = ensure_dtype_nbo(PyArray_DESCR(operands[0])); + if (out_dtypes[0] == NULL) { + return -1; + } /* Make a new NPY_TIMEDELTA, and copy type1's metadata */ out_dtypes[1] = timedelta_dtype_with_copied_meta( PyArray_DESCR(operands[0])); if (out_dtypes[1] == NULL) { + Py_DECREF(out_dtypes[0]); + out_dtypes[0] = NULL; return -1; } - out_dtypes[0] = PyArray_DESCR(operands[0]); - Py_INCREF(out_dtypes[0]); out_dtypes[2] = out_dtypes[0]; Py_INCREF(out_dtypes[2]); @@ -2555,8 +2598,10 @@ PyUFunc_SubtractionTypeResolution(PyUFuncObject *ufunc, else if (PyTypeNum_ISINTEGER(type_num1) || PyTypeNum_ISBOOL(type_num1)) { /* int - m8[] => m8[] - m8[] */ if (type_num2 == NPY_TIMEDELTA) { - out_dtypes[0] = PyArray_DESCR(operands[1]); - Py_INCREF(out_dtypes[0]); + out_dtypes[0] = ensure_dtype_nbo(PyArray_DESCR(operands[1])); + if (out_dtypes[0] == NULL) { + return -1; + } out_dtypes[1] = out_dtypes[0]; Py_INCREF(out_dtypes[1]); out_dtypes[2] = out_dtypes[0]; @@ -2649,8 +2694,10 @@ PyUFunc_MultiplicationTypeResolution(PyUFuncObject *ufunc, if (type_num1 == NPY_TIMEDELTA) { /* m8[] * int## => m8[] * int64 */ if (PyTypeNum_ISINTEGER(type_num2) || PyTypeNum_ISBOOL(type_num2)) { - out_dtypes[0] = PyArray_DESCR(operands[0]); - Py_INCREF(out_dtypes[0]); + out_dtypes[0] = ensure_dtype_nbo(PyArray_DESCR(operands[0])); + if (out_dtypes[0] == NULL) { + return -1; + } out_dtypes[1] = PyArray_DescrNewFromType(NPY_LONGLONG); if (out_dtypes[1] == NULL) { Py_DECREF(out_dtypes[0]); @@ -2664,8 +2711,10 @@ PyUFunc_MultiplicationTypeResolution(PyUFuncObject *ufunc, } /* m8[] * float## => m8[] * float64 */ else if (PyTypeNum_ISFLOAT(type_num2)) { - out_dtypes[0] = PyArray_DESCR(operands[0]); - Py_INCREF(out_dtypes[0]); + out_dtypes[0] = ensure_dtype_nbo(PyArray_DESCR(operands[0])); + if (out_dtypes[0] == NULL) { + return -1; + } out_dtypes[1] = PyArray_DescrNewFromType(NPY_DOUBLE); if (out_dtypes[1] == NULL) { Py_DECREF(out_dtypes[0]); @@ -2688,8 +2737,12 @@ PyUFunc_MultiplicationTypeResolution(PyUFuncObject *ufunc, if (out_dtypes[0] == NULL) { return -1; } - out_dtypes[1] = PyArray_DESCR(operands[1]); - Py_INCREF(out_dtypes[1]); + out_dtypes[1] = ensure_dtype_nbo(PyArray_DESCR(operands[1])); + if (out_dtypes[1] == NULL) { + Py_DECREF(out_dtypes[0]); + out_dtypes[0] = NULL; + return -1; + } out_dtypes[2] = out_dtypes[1]; Py_INCREF(out_dtypes[2]); @@ -2706,8 +2759,12 @@ PyUFunc_MultiplicationTypeResolution(PyUFuncObject *ufunc, if (out_dtypes[0] == NULL) { return -1; } - out_dtypes[1] = PyArray_DESCR(operands[1]); - Py_INCREF(out_dtypes[1]); + out_dtypes[1] = ensure_dtype_nbo(PyArray_DESCR(operands[1])); + if (out_dtypes[1] == NULL) { + Py_DECREF(out_dtypes[0]); + out_dtypes[0] = NULL; + return -1; + } out_dtypes[2] = out_dtypes[1]; Py_INCREF(out_dtypes[2]); @@ -2796,8 +2853,10 @@ PyUFunc_DivisionTypeResolution(PyUFuncObject *ufunc, if (type_num1 == NPY_TIMEDELTA) { /* m8[] / int## => m8[] / int64 */ if (PyTypeNum_ISINTEGER(type_num2)) { - out_dtypes[0] = PyArray_DESCR(operands[0]); - Py_INCREF(out_dtypes[0]); + out_dtypes[0] = ensure_dtype_nbo(PyArray_DESCR(operands[0])); + if (out_dtypes[0] == NULL) { + return -1; + } out_dtypes[1] = PyArray_DescrNewFromType(NPY_LONGLONG); if (out_dtypes[1] == NULL) { Py_DECREF(out_dtypes[0]); @@ -2811,8 +2870,10 @@ PyUFunc_DivisionTypeResolution(PyUFuncObject *ufunc, } /* m8[] / float## => m8[] / float64 */ else if (PyTypeNum_ISFLOAT(type_num2)) { - out_dtypes[0] = PyArray_DESCR(operands[0]); - Py_INCREF(out_dtypes[0]); + out_dtypes[0] = ensure_dtype_nbo(PyArray_DESCR(operands[0])); + if (out_dtypes[0] == NULL) { + return -1; + } out_dtypes[1] = PyArray_DescrNewFromType(NPY_DOUBLE); if (out_dtypes[1] == NULL) { Py_DECREF(out_dtypes[0]); diff --git a/numpy/core/tests/test_ufunc.py b/numpy/core/tests/test_ufunc.py index c75b2d385f6a..578a819c34ff 100644 --- a/numpy/core/tests/test_ufunc.py +++ b/numpy/core/tests/test_ufunc.py @@ -347,6 +347,19 @@ def test_endian(self): a = np.arange(6, dtype=' Date: Mon, 13 Jun 2011 16:59:50 +0200 Subject: [PATCH 33/34] BUG: Py3k: some of the string type-related failures in numpy/core/tests MW: I've removed the asbytes part and changed 'S5' to 'S0' from Derek's original commit. --- numpy/core/tests/test_arrayprint.py | 2 +- numpy/core/tests/test_datetime.py | 30 ++++++++++++++--------------- numpy/core/tests/test_regression.py | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/numpy/core/tests/test_arrayprint.py b/numpy/core/tests/test_arrayprint.py index b1dbb1d68622..5588d244afbf 100644 --- a/numpy/core/tests/test_arrayprint.py +++ b/numpy/core/tests/test_arrayprint.py @@ -82,7 +82,7 @@ def _format_function(x): "[. o O]") assert_(np.array2string(x, formatter={'all':lambda x: "%.4f" % x}) == \ "[0.0000 1.0000 2.0000]") - assert_(np.array2string(x, formatter={'int':lambda x: hex(x)}) == \ + assert_equal(np.array2string(x, formatter={'int':lambda x: hex(x)}), \ "[0x0L 0x1L 0x2L]") x = np.arange(3.) diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index d14b4bc07f28..b51febb8568f 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -116,7 +116,7 @@ def test_datetime_scalar_construction(self): assert_equal(a.dtype, np.dtype('M8[h]')) assert_equal(b.dtype, np.dtype('M8[h]')) - assert_equal(np.datetime64(a), a); + assert_equal(np.datetime64(a), a) assert_equal(np.datetime64(a).dtype, np.dtype('M8[h]')) assert_equal(np.datetime64(b), a) @@ -159,7 +159,7 @@ def test_timedelta_scalar_construction(self): assert_equal(a.dtype, np.dtype('m8[h]')) assert_equal(b.dtype, np.dtype('m8[h]')) - assert_equal(np.timedelta64(a), a); + assert_equal(np.timedelta64(a), a) assert_equal(np.timedelta64(a).dtype, np.dtype('m8[h]')) assert_equal(np.timedelta64(b), a) @@ -312,9 +312,9 @@ def test_days_creation(self): assert_equal(np.array('1601', dtype='M8[D]').astype('i8'), (1600-1970)*365 - (1972-1600)/4 + 3 + 366) assert_equal(np.array('1900', dtype='M8[D]').astype('i8'), - (1900-1970)*365 - (1970-1900)/4) + (1900-1970)*365 - (1970-1900)//4) assert_equal(np.array('1901', dtype='M8[D]').astype('i8'), - (1900-1970)*365 - (1970-1900)/4 + 365) + (1900-1970)*365 - (1970-1900)//4 + 365) assert_equal(np.array('1967', dtype='M8[D]').astype('i8'), -3*365 - 1) assert_equal(np.array('1968', dtype='M8[D]').astype('i8'), -2*365 - 1) assert_equal(np.array('1969', dtype='M8[D]').astype('i8'), -1*365) @@ -324,24 +324,24 @@ def test_days_creation(self): assert_equal(np.array('1973', dtype='M8[D]').astype('i8'), 3*365 + 1) assert_equal(np.array('1974', dtype='M8[D]').astype('i8'), 4*365 + 1) assert_equal(np.array('2000', dtype='M8[D]').astype('i8'), - (2000 - 1970)*365 + (2000 - 1972)/4) + (2000 - 1970)*365 + (2000 - 1972)//4) assert_equal(np.array('2001', dtype='M8[D]').astype('i8'), - (2000 - 1970)*365 + (2000 - 1972)/4 + 366) + (2000 - 1970)*365 + (2000 - 1972)//4 + 366) assert_equal(np.array('2400', dtype='M8[D]').astype('i8'), - (2400 - 1970)*365 + (2400 - 1972)/4 - 3) + (2400 - 1970)*365 + (2400 - 1972)//4 - 3) assert_equal(np.array('2401', dtype='M8[D]').astype('i8'), - (2400 - 1970)*365 + (2400 - 1972)/4 - 3 + 366) + (2400 - 1970)*365 + (2400 - 1972)//4 - 3 + 366) assert_equal(np.array('1600-02-29', dtype='M8[D]').astype('i8'), - (1600-1970)*365 - (1972-1600)/4 + 3 + 31 + 28) + (1600-1970)*365 - (1972-1600)//4 + 3 + 31 + 28) assert_equal(np.array('1600-03-01', dtype='M8[D]').astype('i8'), - (1600-1970)*365 - (1972-1600)/4 + 3 + 31 + 29) + (1600-1970)*365 - (1972-1600)//4 + 3 + 31 + 29) assert_equal(np.array('2000-02-29', dtype='M8[D]').astype('i8'), - (2000 - 1970)*365 + (2000 - 1972)/4 + 31 + 28) + (2000 - 1970)*365 + (2000 - 1972)//4 + 31 + 28) assert_equal(np.array('2000-03-01', dtype='M8[D]').astype('i8'), - (2000 - 1970)*365 + (2000 - 1972)/4 + 31 + 29) + (2000 - 1970)*365 + (2000 - 1972)//4 + 31 + 29) assert_equal(np.array('2001-03-22', dtype='M8[D]').astype('i8'), - (2000 - 1970)*365 + (2000 - 1972)/4 + 366 + 31 + 28 + 21) + (2000 - 1970)*365 + (2000 - 1972)//4 + 366 + 31 + 28 + 21) def test_days_to_pydate(self): assert_equal(np.array('1599', dtype='M8[D]').astype('O'), @@ -1020,7 +1020,7 @@ def test_datetime_as_string(self): date = '1969-12-31T23:59:57.789012345678901234Z' assert_equal(np.datetime_as_string(np.datetime64(date, 'as')), - date); + date) date = '1970-01-01T00:34:56.789012345678901234Z' assert_equal(np.datetime_as_string(np.datetime64(date, 'ns')), @@ -1033,7 +1033,7 @@ def test_datetime_as_string(self): date = '1970-01-01T00:00:05.789012345678901234Z' assert_equal(np.datetime_as_string(np.datetime64(date, 'as')), - date); + date) # String conversion with the unit= parameter a = np.datetime64('2032-07-18T12:23:34.123456Z', 'us') diff --git a/numpy/core/tests/test_regression.py b/numpy/core/tests/test_regression.py index 764bf9a41a67..5db874ecd5f6 100644 --- a/numpy/core/tests/test_regression.py +++ b/numpy/core/tests/test_regression.py @@ -1545,7 +1545,7 @@ def test_string_astype(self): s3 = asbytes('other') a = np.array([[s1],[s2],[s3]]) assert_equal(a.dtype, np.dtype('S5')) - b = a.astype('str') + b = a.astype(np.dtype('S0')) assert_equal(b.dtype, np.dtype('S5')) def test_ticket_1756(self): From 2d7d59aef203ebf25b268ceaccfa1be45237b0df Mon Sep 17 00:00:00 2001 From: Mark Wiebe Date: Thu, 16 Jun 2011 14:49:56 -0500 Subject: [PATCH 34/34] ENH: datetime-ufunc: Add m8 / m8 -> f8 case to the datetime ufunc operations --- numpy/core/code_generators/generate_umath.py | 1 + numpy/core/src/umath/loops.c.src | 15 ++++++++++ numpy/core/src/umath/loops.h | 15 ++++++---- numpy/core/src/umath/loops.h.src | 3 ++ numpy/core/src/umath/ufunc_object.c | 30 +++++++++++++++++--- numpy/core/tests/test_datetime.py | 16 +++++++---- 6 files changed, 65 insertions(+), 15 deletions(-) diff --git a/numpy/core/code_generators/generate_umath.py b/numpy/core/code_generators/generate_umath.py index 599ffddc8bf2..957607f40113 100644 --- a/numpy/core/code_generators/generate_umath.py +++ b/numpy/core/code_generators/generate_umath.py @@ -275,6 +275,7 @@ def english_upper(s): TD(intfltcmplx), [TypeDescription('m', FullTypeDescr, 'mq', 'm'), TypeDescription('m', FullTypeDescr, 'md', 'm'), + TypeDescription('m', FullTypeDescr, 'mm', 'd'), ], TD(O, f='PyNumber_Divide'), ), diff --git a/numpy/core/src/umath/loops.c.src b/numpy/core/src/umath/loops.c.src index 31345ff4539c..b23f1018c815 100644 --- a/numpy/core/src/umath/loops.c.src +++ b/numpy/core/src/umath/loops.c.src @@ -1229,6 +1229,21 @@ TIMEDELTA_md_m_divide(char **args, intp *dimensions, intp *steps, void *NPY_UNUS } } +NPY_NO_EXPORT void +TIMEDELTA_mm_d_divide(char **args, intp *dimensions, intp *steps, void *NPY_UNUSED(func)) +{ + BINARY_LOOP { + const npy_timedelta in1 = *(npy_timedelta *)ip1; + const npy_timedelta in2 = *(npy_timedelta *)ip2; + if (in1 == NPY_DATETIME_NAT || in2 == NPY_DATETIME_NAT) { + *((double *)op1) = NPY_NAN; + } + else { + *((double *)op1) = (double)in1 / (double)in2; + } + } +} + /* ***************************************************************************** ** FLOAT LOOPS ** diff --git a/numpy/core/src/umath/loops.h b/numpy/core/src/umath/loops.h index 2a827e168aa0..2a792bf5b429 100644 --- a/numpy/core/src/umath/loops.h +++ b/numpy/core/src/umath/loops.h @@ -2661,6 +2661,9 @@ TIMEDELTA_mq_m_divide(char **args, intp *dimensions, intp *steps, void *NPY_UNUS NPY_NO_EXPORT void TIMEDELTA_md_m_divide(char **args, intp *dimensions, intp *steps, void *NPY_UNUSED(func)); +NPY_NO_EXPORT void +TIMEDELTA_mm_d_divide(char **args, intp *dimensions, intp *steps, void *NPY_UNUSED(func)); + /* Special case equivalents to above functions */ #define TIMEDELTA_mq_m_true_divide TIMEDELTA_mq_m_divide @@ -2678,27 +2681,27 @@ TIMEDELTA_md_m_divide(char **args, intp *dimensions, intp *steps, void *NPY_UNUS ***************************************************************************** */ -#line 511 +#line 514 NPY_NO_EXPORT void OBJECT_equal(char **args, intp *dimensions, intp *steps, void *NPY_UNUSED(func)); -#line 511 +#line 514 NPY_NO_EXPORT void OBJECT_not_equal(char **args, intp *dimensions, intp *steps, void *NPY_UNUSED(func)); -#line 511 +#line 514 NPY_NO_EXPORT void OBJECT_greater(char **args, intp *dimensions, intp *steps, void *NPY_UNUSED(func)); -#line 511 +#line 514 NPY_NO_EXPORT void OBJECT_greater_equal(char **args, intp *dimensions, intp *steps, void *NPY_UNUSED(func)); -#line 511 +#line 514 NPY_NO_EXPORT void OBJECT_less(char **args, intp *dimensions, intp *steps, void *NPY_UNUSED(func)); -#line 511 +#line 514 NPY_NO_EXPORT void OBJECT_less_equal(char **args, intp *dimensions, intp *steps, void *NPY_UNUSED(func)); diff --git a/numpy/core/src/umath/loops.h.src b/numpy/core/src/umath/loops.h.src index bf41e41871f3..fdedc1933390 100644 --- a/numpy/core/src/umath/loops.h.src +++ b/numpy/core/src/umath/loops.h.src @@ -487,6 +487,9 @@ TIMEDELTA_mq_m_divide(char **args, intp *dimensions, intp *steps, void *NPY_UNUS NPY_NO_EXPORT void TIMEDELTA_md_m_divide(char **args, intp *dimensions, intp *steps, void *NPY_UNUSED(func)); +NPY_NO_EXPORT void +TIMEDELTA_mm_d_divide(char **args, intp *dimensions, intp *steps, void *NPY_UNUSED(func)); + /* Special case equivalents to above functions */ #define TIMEDELTA_mq_m_true_divide TIMEDELTA_mq_m_divide diff --git a/numpy/core/src/umath/ufunc_object.c b/numpy/core/src/umath/ufunc_object.c index 2e46fc51aeda..dcc29f2ba797 100644 --- a/numpy/core/src/umath/ufunc_object.c +++ b/numpy/core/src/umath/ufunc_object.c @@ -2822,8 +2822,9 @@ type_reso_error: { /* * This function applies the type resolution rules for division. * In particular, there are a number of special cases with datetime: - * m8[] / int## => m8[] / int64 - * m8[] / float## => m8[] / float64 + * m8[] / m8[] to m8[gcd(,)] / m8[gcd(,)] -> float64 + * m8[] / int## to m8[] / int64 -> m8[] + * m8[] / float## to m8[] / float64 -> m8[] */ NPY_NO_EXPORT int PyUFunc_DivisionTypeResolution(PyUFuncObject *ufunc, @@ -2851,13 +2852,34 @@ PyUFunc_DivisionTypeResolution(PyUFuncObject *ufunc, } if (type_num1 == NPY_TIMEDELTA) { + /* + * m8[] / m8[] to + * m8[gcd(,)] / m8[gcd(,)] -> float64 + */ + if (type_num2 == NPY_TIMEDELTA) { + out_dtypes[0] = PyArray_PromoteTypes(PyArray_DESCR(operands[0]), + PyArray_DESCR(operands[1])); + if (out_dtypes[0] == NULL) { + return -1; + } + out_dtypes[1] = out_dtypes[0]; + Py_INCREF(out_dtypes[1]); + out_dtypes[2] = PyArray_DescrFromType(NPY_DOUBLE); + if (out_dtypes[2] == NULL) { + Py_DECREF(out_dtypes[0]); + out_dtypes[0] = NULL; + Py_DECREF(out_dtypes[1]); + out_dtypes[1] = NULL; + return -1; + } + } /* m8[] / int## => m8[] / int64 */ - if (PyTypeNum_ISINTEGER(type_num2)) { + else if (PyTypeNum_ISINTEGER(type_num2)) { out_dtypes[0] = ensure_dtype_nbo(PyArray_DESCR(operands[0])); if (out_dtypes[0] == NULL) { return -1; } - out_dtypes[1] = PyArray_DescrNewFromType(NPY_LONGLONG); + out_dtypes[1] = PyArray_DescrFromType(NPY_LONGLONG); if (out_dtypes[1] == NULL) { Py_DECREF(out_dtypes[0]); out_dtypes[0] = NULL; diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index b51febb8568f..c533f3b2c2e3 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -738,31 +738,37 @@ def test_datetime_multiply(self): assert_raises(TypeError, np.multiply, 1.5, dta) def test_datetime_divide(self): - for dta, tda, tdb, tdc in \ + for dta, tda, tdb, tdc, tdd in \ [ # One-dimensional arrays (np.array(['2012-12-21'], dtype='M8[D]'), np.array([6], dtype='m8[h]'), np.array([9], dtype='m8[h]'), - np.array([12], dtype='m8[h]')), + np.array([12], dtype='m8[h]'), + np.array([6], dtype='m8[m]')), # NumPy scalars (np.datetime64('2012-12-21', '[D]'), np.timedelta64(6, '[h]'), np.timedelta64(9, '[h]'), - np.timedelta64(12, '[h]'))]: + np.timedelta64(12, '[h]'), + np.timedelta64(6, '[m]'))]: # m8 / int assert_equal(tdc / 2, tda) assert_equal((tdc / 2).dtype, np.dtype('m8[h]')) # m8 / float assert_equal(tda / 0.5, tdc) assert_equal((tda / 0.5).dtype, np.dtype('m8[h]')) + # m8 / m8 + assert_equal(tda / tdb, 6.0 / 9.0) + assert_equal(tdb / tda, 9.0 / 6.0) + assert_equal((tda / tdb).dtype, np.dtype('f8')) + assert_equal(tda / tdd, 60.0) + assert_equal(tdd / tda, 1.0 / 60.0) # int / m8 assert_raises(TypeError, np.divide, 2, tdb) # float / m8 assert_raises(TypeError, np.divide, 0.5, tdb) - # m8 / m8 - assert_raises(TypeError, np.divide, tda, tdb) # m8 / M8 assert_raises(TypeError, np.divide, dta, tda) # M8 / m8