diff options
author | Charles Harris <charlesr.harris@gmail.com> | 2012-05-10 01:43:30 -0600 |
---|---|---|
committer | Charles Harris <charlesr.harris@gmail.com> | 2012-05-10 01:43:30 -0600 |
commit | bb162a291b765a34a2e6fab845c36f34c01357af (patch) | |
tree | a339c52a8f7b4b0bd9c2725b090231c18f9be24b | |
parent | c869d124d08665b703539c9b2f14ec6ffbae0be0 (diff) | |
parent | ddc944eb6ae67de5c0e184505651ca1f50517bbd (diff) | |
download | numpy-bb162a291b765a34a2e6fab845c36f34c01357af.tar.gz |
Merge branch 'pull-274'
* pull-274:
BUG: Fix datetime 1.6 pickle compatibility test for Python 3.
STY: Elaborate comment about cloning dtype's c_metadata
DOC: Improve documentation comment from PR 274 comment
ENH: Change NPY_AUXDATA_FREE macro based on PR feedback
BUG: Output the datetime64 dtype to the pickle format from 1.6
ENH: Change datetime64 to use c_metadata instead of metadata
ENH: Add a NpyAuxData c_metadata to PyArray_Descr
-rw-r--r-- | numpy/core/_internal.py | 3 | ||||
-rw-r--r-- | numpy/core/include/numpy/ndarraytypes.h | 202 | ||||
-rw-r--r-- | numpy/core/include/numpy/npy_deprecated_api.h | 24 | ||||
-rw-r--r-- | numpy/core/src/multiarray/_datetime.h | 30 | ||||
-rw-r--r-- | numpy/core/src/multiarray/arraytypes.c.src | 119 | ||||
-rw-r--r-- | numpy/core/src/multiarray/datetime.c | 310 | ||||
-rw-r--r-- | numpy/core/src/multiarray/datetime_strings.c | 5 | ||||
-rw-r--r-- | numpy/core/src/multiarray/datetime_strings.h | 2 | ||||
-rw-r--r-- | numpy/core/src/multiarray/descriptor.c | 251 | ||||
-rw-r--r-- | numpy/core/src/multiarray/multiarraymodule.c | 4 | ||||
-rw-r--r-- | numpy/core/src/multiarray/scalarapi.c | 32 | ||||
-rw-r--r-- | numpy/core/src/umath/ufunc_type_resolution.c | 57 | ||||
-rw-r--r-- | numpy/core/tests/test_datetime.py | 18 | ||||
-rw-r--r-- | numpy/lib/type_check.py | 1 |
14 files changed, 469 insertions, 589 deletions
diff --git a/numpy/core/_internal.py b/numpy/core/_internal.py index 3d6702095..309b53c44 100644 --- a/numpy/core/_internal.py +++ b/numpy/core/_internal.py @@ -80,7 +80,6 @@ def _usefields(adict, align): # a simple typestring def _array_descr(descriptor): - from multiarray import METADATA_DTSTR fields = descriptor.fields if fields is None: subdtype = descriptor.subdtype @@ -89,8 +88,6 @@ def _array_descr(descriptor): return descriptor.str else: new = descriptor.metadata.copy() - # Eliminate any key related to internal implementation - new.pop(METADATA_DTSTR, None) if new: return (descriptor.str, new) else: diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h index ec890f87d..db5257761 100644 --- a/numpy/core/include/numpy/ndarraytypes.h +++ b/numpy/core/include/numpy/ndarraytypes.h @@ -1,7 +1,7 @@ #ifndef NDARRAYTYPES_H #define NDARRAYTYPES_H -/* This is auto-generated by the installer */ +/* numpyconfig.h is auto-generated by the installer */ #include "numpyconfig.h" #include "npy_common.h" @@ -85,8 +85,6 @@ enum NPY_TYPES { NPY_BOOL=0, NPY_NTYPES_ABI_COMPATIBLE=21 }; -#define NPY_METADATA_DTSTR "__timeunit__" - /* basetype array priority */ #define NPY_PRIORITY 0.0 @@ -215,6 +213,7 @@ typedef enum { /* The special not-a-time (NaT) value */ #define NPY_DATETIME_NAT NPY_MIN_INT64 + /* * Upper bound on the length of a DATETIME ISO 8601 string * YEAR: 21 (64-bit year) @@ -249,20 +248,6 @@ typedef enum { #define NPY_DATETIME_NUMUNITS (NPY_FR_GENERIC + 1) #define NPY_DATETIME_DEFAULTUNIT NPY_FR_GENERIC -#define NPY_STR_Y "Y" -#define NPY_STR_M "M" -#define NPY_STR_W "W" -#define NPY_STR_D "D" -#define NPY_STR_h "h" -#define NPY_STR_m "m" -#define NPY_STR_s "s" -#define NPY_STR_ms "ms" -#define NPY_STR_us "us" -#define NPY_STR_ns "ns" -#define NPY_STR_ps "ps" -#define NPY_STR_fs "fs" -#define NPY_STR_as "as" - /* * Business day conventions for mapping invalid business * days to valid business days. @@ -290,6 +275,42 @@ typedef enum { NPY_BUSDAY_RAISE } NPY_BUSDAY_ROLL; +/************************************************************ + * NumPy Auxiliary Data for inner loops, sort functions, etc. + ************************************************************/ + +/* + * When creating an auxiliary data struct, this should always appear + * as the first member, like this: + * + * typedef struct { + * NpyAuxData base; + * double constant; + * } constant_multiplier_aux_data; + */ +typedef struct NpyAuxData_tag NpyAuxData; + +/* Function pointers for freeing or cloning auxiliary data */ +typedef void (NpyAuxData_FreeFunc) (NpyAuxData *); +typedef NpyAuxData *(NpyAuxData_CloneFunc) (NpyAuxData *); + +struct NpyAuxData_tag { + NpyAuxData_FreeFunc *free; + NpyAuxData_CloneFunc *clone; + /* To allow for a bit of expansion without breaking the ABI */ + void *reserved[2]; +}; + +/* Macros to use for freeing and cloning auxiliary data */ +#define NPY_AUXDATA_FREE(auxdata) \ + do { \ + if ((auxdata) != NULL) { \ + (auxdata)->free(auxdata); \ + } \ + } while(0) +#define NPY_AUXDATA_CLONE(auxdata) \ + ((auxdata)->clone(auxdata)) + /********************************************************************* * NumPy functions for dealing with masks, such as in masked iteration *********************************************************************/ @@ -571,44 +592,60 @@ typedef struct { typedef struct _PyArray_Descr { PyObject_HEAD - PyTypeObject *typeobj; /* - * the type object representing an - * instance of this type -- should not - * be two type_numbers with the same type - * object. - */ - char kind; /* kind for this type */ - char type; /* unique-character representing this type */ - char byteorder; /* - * '>' (big), '<' (little), '|' - * (not-applicable), or '=' (native). - */ - char flags; /* flags describing data type */ - int type_num; /* number representing this type */ - int elsize; /* element size for this type */ - int alignment; /* alignment needed for this type */ - struct _arr_descr \ - *subarray; /* - * Non-NULL if this type is - * is an array (C-contiguous) - * of some other type - */ - PyObject *fields; /* The fields dictionary for this type - * For statically defined descr this - * is always Py_None - */ - - PyObject *names; /* - * An ordered tuple of field names or NULL - * if no fields are defined - */ - - PyArray_ArrFuncs *f; /* - * a table of functions specific for each - * basic data descriptor - */ - - PyObject *metadata; /* Metadata about this dtype */ + /* + * the type object representing an + * instance of this type -- should not + * be two type_numbers with the same type + * object. + */ + PyTypeObject *typeobj; + /* kind for this type */ + char kind; + /* unique-character representing this type */ + char type; + /* + * '>' (big), '<' (little), '|' + * (not-applicable), or '=' (native). + */ + char byteorder; + /* flags describing data type */ + char flags; + /* number representing this type */ + int type_num; + /* element size (itemsize) for this type */ + int elsize; + /* alignment needed for this type */ + int alignment; + /* + * Non-NULL if this type is + * is an array (C-contiguous) + * of some other type + */ + struct _arr_descr *subarray; + /* + * The fields dictionary for this type + * For statically defined descr this + * is always Py_None + */ + PyObject *fields; + /* + * An ordered tuple of field names or NULL + * if no fields are defined + */ + PyObject *names; + /* + * a table of functions specific for each + * basic data descriptor + */ + PyArray_ArrFuncs *f; + /* Metadata about this dtype */ + PyObject *metadata; + /* + * Metadata specific to the C implementation + * of the particular dtype. This was added + * for NumPy 1.7.0. + */ + NpyAuxData *c_metadata; } PyArray_Descr; typedef struct _arr_descr { @@ -724,20 +761,16 @@ typedef struct { int flags; } PyArray_Chunk; - typedef struct { - NPY_DATETIMEUNIT base; - int num; - /* - * 'den' and 'events are unused, kept here for ABI - * compatibility with 1.6. - * - * TODO: Remove for 2.0. - */ - int den; - int events; + NPY_DATETIMEUNIT base; + int num; } PyArray_DatetimeMetaData; +typedef struct { + NpyAuxData base; + PyArray_DatetimeMetaData meta; +} PyArray_DatetimeDTypeMetaData; + /* * This structure contains an exploded view of a date-time value. * NaT is represented by year == NPY_DATETIME_NAT. @@ -747,7 +780,7 @@ typedef struct { npy_int32 month, day, hour, min, sec, us, ps, as; } npy_datetimestruct; -/* TO BE REMOVED - NOT USED INTERNALLY. */ +/* This is not used internally. */ typedef struct { npy_int64 day; npy_int32 sec, us, ps, as; @@ -1748,41 +1781,6 @@ PyArray_HASMASKNA(PyArrayObject *arr) #define PyDataType_ISBYTESWAPPED(d) (!PyDataType_ISNOTSWAPPED(d)) /************************************************************ - * NumPy Auxiliary Data for inner loops, sort functions, etc. - ************************************************************/ - -/* - * When creating an auxiliary data struct, this should always appear - * as the first member, like this: - * - * typedef struct { - * NpyAuxData base; - * double constant; - * } constant_multiplier_aux_data; - */ -typedef struct NpyAuxData_tag NpyAuxData; - -/* Function pointers for freeing or cloning auxiliary data */ -typedef void (NpyAuxData_FreeFunc) (NpyAuxData *); -typedef NpyAuxData *(NpyAuxData_CloneFunc) (NpyAuxData *); - -struct NpyAuxData_tag { - NpyAuxData_FreeFunc *free; - NpyAuxData_CloneFunc *clone; - /* To allow for a bit of expansion without breaking the ABI */ - void *reserved[2]; -}; - -/* Macros to use for freeing and cloning auxiliary data */ -#define NPY_AUXDATA_FREE(auxdata) \ - if ((auxdata) == NULL) \ - ; \ - else \ - ((auxdata)->free(auxdata)) -#define NPY_AUXDATA_CLONE(auxdata) \ - ((auxdata)->clone(auxdata)) - -/************************************************************ * A struct used by PyArray_CreateSortedStridePerm, new in 1.7. ************************************************************/ diff --git a/numpy/core/include/numpy/npy_deprecated_api.h b/numpy/core/include/numpy/npy_deprecated_api.h index 2fab2cd0d..f082d0827 100644 --- a/numpy/core/include/numpy/npy_deprecated_api.h +++ b/numpy/core/include/numpy/npy_deprecated_api.h @@ -95,6 +95,30 @@ */ #define FORTRAN_IF PyArray_FORTRAN_IF +/* Deprecated as of NumPy 1.7, datetime64 uses c_metadata instead */ +#define NPY_METADATA_DTSTR "__timeunit__" + +/* + * Deprecated as of NumPy 1.7. + * The reasoning: + * - These are for datetime, but there's no datetime "namespace". + * - They just turn NPY_STR_<x> into "<x>", which is just + * making something simple be indirected. + */ +#define NPY_STR_Y "Y" +#define NPY_STR_M "M" +#define NPY_STR_W "W" +#define NPY_STR_D "D" +#define NPY_STR_h "h" +#define NPY_STR_m "m" +#define NPY_STR_s "s" +#define NPY_STR_ms "ms" +#define NPY_STR_us "us" +#define NPY_STR_ns "ns" +#define NPY_STR_ps "ps" +#define NPY_STR_fs "fs" +#define NPY_STR_as "as" + /* * The macros in old_defines.h are Deprecated as of NumPy 1.7 and will be * removed in the next major release. diff --git a/numpy/core/src/multiarray/_datetime.h b/numpy/core/src/multiarray/_datetime.h index 3c216d856..b359a9f17 100644 --- a/numpy/core/src/multiarray/_datetime.h +++ b/numpy/core/src/multiarray/_datetime.h @@ -33,13 +33,6 @@ NPY_NO_EXPORT PyArray_Descr * create_datetime_dtype_with_unit(int type_num, NPY_DATETIMEUNIT unit); /* - * This function returns the a new reference to the - * capsule with the datetime metadata. - */ -NPY_NO_EXPORT PyObject * -get_datetime_metacobj_from_dtype(PyArray_Descr *dtype); - -/* * This function returns a pointer to the DateTimeMetaData * contained within the provided datetime dtype. */ @@ -158,20 +151,6 @@ can_cast_timedelta64_metadata(PyArray_DatetimeMetaData *src_meta, NPY_CASTING casting); /* - * Computes the GCD of the two date-time metadata values. Raises - * an exception if there is no reasonable GCD, such as with - * years and days. - * - * Returns a capsule with the GCD metadata. - */ -NPY_NO_EXPORT PyObject * -compute_datetime_metadata_greatest_common_divisor_capsule( - PyArray_Descr *type1, - PyArray_Descr *type2, - int strict_with_nonlinear_units1, - int strict_with_nonlinear_units2); - -/* * Computes the conversion factor to convert data with 'src_meta' metadata * into data with 'dst_meta' metadata. * @@ -184,7 +163,7 @@ get_datetime_conversion_factor(PyArray_DatetimeMetaData *src_meta, npy_int64 *out_num, npy_int64 *out_denom); /* - * Given an the capsule datetime metadata object, + * Given a pointer to datetime metadata, * returns a tuple for pickling and other purposes. */ NPY_NO_EXPORT PyObject * @@ -200,13 +179,6 @@ convert_datetime_metadata_tuple_to_datetime_metadata(PyObject *tuple, PyArray_DatetimeMetaData *out_meta); /* - * Given a tuple representing datetime metadata, - * returns a capsule datetime metadata object. - */ -NPY_NO_EXPORT PyObject * -convert_datetime_metadata_tuple_to_metacobj(PyObject *tuple); - -/* * Gets a tzoffset in minutes by calling the fromutc() function on * the Python datetime.tzinfo object. */ diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src index 9dbcb8439..6dcc7d8e9 100644 --- a/numpy/core/src/multiarray/arraytypes.c.src +++ b/numpy/core/src/multiarray/arraytypes.c.src @@ -3699,18 +3699,33 @@ static PyArray_ArrFuncs _Py@NAME@_ArrFuncs = { */ static PyArray_Descr @from@_Descr = { PyObject_HEAD_INIT(&PyArrayDescr_Type) + /* typeobj */ &Py@NAME@ArrType_Type, + /* kind */ NPY_@from@LTR, + /* type */ NPY_@from@LTR, + /* byteorder */ '@endian@', + /* flags */ 0, + /* type_num */ NPY_@from@, + /* elsize */ 0, + /* alignment */ _ALIGN(@align@), + /* subarray */ NULL, + /* fields */ NULL, + /* names */ NULL, + /* f */ &_Py@NAME@_ArrFuncs, + /* metadata */ + NULL, + /* c_metadata */ NULL, }; @@ -3824,44 +3839,38 @@ static PyArray_ArrFuncs _Py@NAME@_ArrFuncs = { */ NPY_NO_EXPORT PyArray_Descr @from@_Descr = { PyObject_HEAD_INIT(&PyArrayDescr_Type) + /* typeobj */ &Py@NAME@ArrType_Type, + /* kind */ NPY_@kind@LTR, + /* type */ NPY_@from@LTR, + /* byteorder */ '@endian@', + /* flags */ @isobject@, + /* type_num */ NPY_@from@, + /* elsize */ @num@*sizeof(@fromtype@), + /* alignment */ _ALIGN(@fromtype@), + /* subarray */ NULL, + /* fields */ NULL, + /* names */ NULL, + /* f */ &_Py@NAME@_ArrFuncs, + /* metadata */ + NULL, + /* c_metadata */ NULL, }; /**end repeat**/ -static void -_init_datetime_descr(PyArray_Descr *descr) -{ - PyArray_DatetimeMetaData *dt_data; - PyObject *cobj; - - dt_data = PyArray_malloc(sizeof(PyArray_DatetimeMetaData)); - dt_data->base = NPY_DATETIME_DEFAULTUNIT; - dt_data->num = 1; - -/* FIXME - * There is no error check here and no way to indicate an error - * until the metadata turns up NULL. - */ - cobj = NpyCapsule_FromVoidPtr((void *)dt_data, simple_capsule_dtor); - descr->metadata = PyDict_New(); - PyDict_SetItemString(descr->metadata, NPY_METADATA_DTSTR, cobj); - Py_DECREF(cobj); - -} - #define _MAX_LETTER 128 static char _letter_to_num[_MAX_LETTER]; @@ -3943,16 +3952,64 @@ PyArray_DescrFromType(int type) Py_INCREF(ret); } - /* Make sure dtype metadata is initialized for DATETIME */ - if (PyTypeNum_ISDATETIME(type)) { - if (ret->metadata == NULL) { - _init_datetime_descr(ret); - } + return ret; +} + +/* A clone function for the datetime dtype metadata */ +static NpyAuxData * +datetime_dtype_metadata_clone(NpyAuxData *data) +{ + PyArray_DatetimeDTypeMetaData *newdata = + (PyArray_DatetimeDTypeMetaData *)PyArray_malloc( + sizeof(PyArray_DatetimeDTypeMetaData)); + if (newdata == NULL) { + return NULL; } - return ret; + memcpy(newdata, data, sizeof(PyArray_DatetimeDTypeMetaData)); + + return (NpyAuxData *)newdata; } +/* + * Initializes the c_metadata field for the _builtin_descrs DATETIME + * and TIMEDELTA. + */ +int +initialize_builtin_datetime_metadata(void) +{ + PyArray_DatetimeDTypeMetaData *data1, *data2; + + /* Allocate memory for the metadata */ + data1 = PyArray_malloc(sizeof(PyArray_DatetimeDTypeMetaData)); + if (data1 == NULL) { + return -1; + } + data2 = PyArray_malloc(sizeof(PyArray_DatetimeDTypeMetaData)); + if (data2 == NULL) { + PyArray_free(data1); + return -1; + } + + /* Initialize the base aux data */ + memset(data1, 0, sizeof(PyArray_DatetimeDTypeMetaData)); + memset(data2, 0, sizeof(PyArray_DatetimeDTypeMetaData)); + data1->base.free = (NpyAuxData_FreeFunc *)PyArray_free; + data2->base.free = (NpyAuxData_FreeFunc *)PyArray_free; + data1->base.clone = datetime_dtype_metadata_clone; + data2->base.clone = datetime_dtype_metadata_clone; + + /* Set to the default metadata */ + data1->meta.base = NPY_DATETIME_DEFAULTUNIT; + data1->meta.num = 1; + data2->meta.base = NPY_DATETIME_DEFAULTUNIT; + data2->meta.num = 1; + + _builtin_descrs[NPY_DATETIME]->c_metadata = (NpyAuxData *)data1; + _builtin_descrs[NPY_TIMEDELTA]->c_metadata = (NpyAuxData *)data1; + + return 0; +} /* ***************************************************************************** @@ -3961,6 +4018,10 @@ PyArray_DescrFromType(int type) */ +/* + * This function is called during numpy module initialization, + * and is used to initialize internal dtype tables. + */ NPY_NO_EXPORT int set_typeinfo(PyObject *dict) { @@ -4018,6 +4079,10 @@ set_typeinfo(PyObject *dict) /**end repeat**/ + if (initialize_builtin_datetime_metadata() < 0) { + return -1; + } + for (i = 0; i < _MAX_LETTER; i++) { _letter_to_num[i] = NPY_NTYPES; } diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c index 08cbb9e04..6a209635f 100644 --- a/numpy/core/src/multiarray/datetime.c +++ b/numpy/core/src/multiarray/datetime.c @@ -31,26 +31,26 @@ * This is called during module initialization */ NPY_NO_EXPORT void -numpy_pydatetime_import() +numpy_pydatetime_import(void) { PyDateTime_IMPORT; } /* Exported as DATETIMEUNITS in multiarraymodule.c */ NPY_NO_EXPORT char *_datetime_strings[NPY_DATETIME_NUMUNITS] = { - NPY_STR_Y, - NPY_STR_M, - NPY_STR_W, - NPY_STR_D, - NPY_STR_h, - NPY_STR_m, - NPY_STR_s, - NPY_STR_ms, - NPY_STR_us, - NPY_STR_ns, - NPY_STR_ps, - NPY_STR_fs, - NPY_STR_as, + "Y", + "M", + "W", + "D", + "h", + "m", + "s", + "ms", + "us", + "ns", + "ps", + "fs", + "as", "generic" }; @@ -227,7 +227,7 @@ set_datetimestruct_days(npy_int64 days, npy_datetimestruct *dts) for (i = 0; i < 12; ++i) { if (days < month_lengths[i]) { dts->month = i + 1; - dts->day = days + 1; + dts->day = (int)days + 1; return; } else { @@ -480,7 +480,7 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, set_datetimestruct_days((dt - (perday-1)) / perday, out); dt = (perday-1) + (dt + 1) % perday; } - out->hour = dt; + out->hour = (int)dt; break; case NPY_FR_m: @@ -494,8 +494,8 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, set_datetimestruct_days((dt - (perday-1)) / perday, out); dt = (perday-1) + (dt + 1) % perday; } - out->hour = dt / 60; - out->min = dt % 60; + out->hour = (int)(dt / 60); + out->min = (int)(dt % 60); break; case NPY_FR_s: @@ -509,9 +509,9 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, set_datetimestruct_days((dt - (perday-1)) / perday, out); dt = (perday-1) + (dt + 1) % perday; } - out->hour = dt / (60*60); - out->min = (dt / 60) % 60; - out->sec = dt % 60; + out->hour = (int)(dt / (60*60)); + out->min = (int)((dt / 60) % 60); + out->sec = (int)(dt % 60); break; case NPY_FR_ms: @@ -525,10 +525,10 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, set_datetimestruct_days((dt - (perday-1)) / perday, out); dt = (perday-1) + (dt + 1) % perday; } - out->hour = dt / (60*60*1000LL); - out->min = (dt / (60*1000LL)) % 60; - out->sec = (dt / 1000LL) % 60; - out->us = (dt % 1000LL) * 1000; + out->hour = (int)(dt / (60*60*1000LL)); + out->min = (int)((dt / (60*1000LL)) % 60); + out->sec = (int)((dt / 1000LL) % 60); + out->us = (int)((dt % 1000LL) * 1000); break; case NPY_FR_us: @@ -542,10 +542,10 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, set_datetimestruct_days((dt - (perday-1)) / perday, out); dt = (perday-1) + (dt + 1) % perday; } - out->hour = dt / (60*60*1000000LL); - out->min = (dt / (60*1000000LL)) % 60; - out->sec = (dt / 1000000LL) % 60; - out->us = dt % 1000000LL; + out->hour = (int)(dt / (60*60*1000000LL)); + out->min = (int)((dt / (60*1000000LL)) % 60); + out->sec = (int)((dt / 1000000LL) % 60); + out->us = (int)(dt % 1000000LL); break; case NPY_FR_ns: @@ -559,11 +559,11 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, set_datetimestruct_days((dt - (perday-1)) / perday, out); dt = (perday-1) + (dt + 1) % perday; } - out->hour = dt / (60*60*1000000000LL); - out->min = (dt / (60*1000000000LL)) % 60; - out->sec = (dt / 1000000000LL) % 60; - out->us = (dt / 1000LL) % 1000000LL; - out->ps = (dt % 1000LL) * 1000; + out->hour = (int)(dt / (60*60*1000000000LL)); + out->min = (int)((dt / (60*1000000000LL)) % 60); + out->sec = (int)((dt / 1000000000LL) % 60); + out->us = (int)((dt / 1000LL) % 1000000LL); + out->ps = (int)((dt % 1000LL) * 1000); break; case NPY_FR_ps: @@ -577,22 +577,22 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, set_datetimestruct_days((dt - (perday-1)) / perday, out); dt = (perday-1) + (dt + 1) % perday; } - out->hour = dt / (60*60*1000000000000LL); - out->min = (dt / (60*1000000000000LL)) % 60; - out->sec = (dt / 1000000000000LL) % 60; - out->us = (dt / 1000000LL) % 1000000LL; - out->ps = dt % 1000000LL; + out->hour = (int)(dt / (60*60*1000000000000LL)); + out->min = (int)((dt / (60*1000000000000LL)) % 60); + out->sec = (int)((dt / 1000000000000LL) % 60); + out->us = (int)((dt / 1000000LL) % 1000000LL); + out->ps = (int)(dt % 1000000LL); break; case NPY_FR_fs: /* entire range is only +- 2.6 hours */ if (dt >= 0) { - out->hour = dt / (60*60*1000000000000000LL); - out->min = (dt / (60*1000000000000000LL)) % 60; - out->sec = (dt / 1000000000000000LL) % 60; - out->us = (dt / 1000000000LL) % 1000000LL; - out->ps = (dt / 1000LL) % 1000000LL; - out->as = (dt % 1000LL) * 1000; + out->hour = (int)(dt / (60*60*1000000000000000LL)); + out->min = (int)((dt / (60*1000000000000000LL)) % 60); + out->sec = (int)((dt / 1000000000000000LL) % 60); + out->us = (int)((dt / 1000000000LL) % 1000000LL); + out->ps = (int)((dt / 1000LL) % 1000000LL); + out->as = (int)((dt % 1000LL) * 1000); } else { npy_datetime minutes; @@ -605,20 +605,20 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, } /* Offset the negative minutes */ add_minutes_to_datetimestruct(out, minutes); - out->sec = (dt / 1000000000000000LL) % 60; - out->us = (dt / 1000000000LL) % 1000000LL; - out->ps = (dt / 1000LL) % 1000000LL; - out->as = (dt % 1000LL) * 1000; + out->sec = (int)((dt / 1000000000000000LL) % 60); + out->us = (int)((dt / 1000000000LL) % 1000000LL); + out->ps = (int)((dt / 1000LL) % 1000000LL); + out->as = (int)((dt % 1000LL) * 1000); } break; case NPY_FR_as: /* entire range is only +- 9.2 seconds */ if (dt >= 0) { - out->sec = (dt / 1000000000000000000LL) % 60; - out->us = (dt / 1000000000000LL) % 1000000LL; - out->ps = (dt / 1000000LL) % 1000000LL; - out->as = dt % 1000000LL; + out->sec = (int)((dt / 1000000000000000000LL) % 60); + out->us = (int)((dt / 1000000000000LL) % 1000000LL); + out->ps = (int)((dt / 1000000LL) % 1000000LL); + out->as = (int)(dt % 1000000LL); } else { npy_datetime seconds; @@ -631,9 +631,9 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta, } /* Offset the negative seconds */ add_seconds_to_datetimestruct(out, seconds); - out->us = (dt / 1000000000000LL) % 1000000LL; - out->ps = (dt / 1000000LL) % 1000000LL; - out->as = dt % 1000000LL; + out->us = (int)((dt / 1000000000000LL) % 1000000LL); + out->ps = (int)((dt / 1000000LL) % 1000000LL); + out->as = (int)(dt % 1000000LL); } break; @@ -692,7 +692,6 @@ create_datetime_dtype(int type_num, PyArray_DatetimeMetaData *meta) { PyArray_Descr *dtype = NULL; PyArray_DatetimeMetaData *dt_data; - PyObject *metacobj = NULL; /* Create a default datetime or timedelta */ if (type_num == NPY_DATETIME || type_num == NPY_TIMEDELTA) { @@ -709,44 +708,11 @@ create_datetime_dtype(int type_num, PyArray_DatetimeMetaData *meta) return NULL; } - /* - * Remove any reference to old metadata dictionary - * And create a new one for this new dtype - */ - Py_XDECREF(dtype->metadata); - dtype->metadata = PyDict_New(); - if (dtype->metadata == NULL) { - Py_DECREF(dtype); - return NULL; - } - - /* Create a metadata capsule to copy the provided metadata */ - dt_data = PyArray_malloc(sizeof(PyArray_DatetimeMetaData)); - if (dt_data == NULL) { - Py_DECREF(dtype); - PyErr_NoMemory(); - return NULL; - } + dt_data = &(((PyArray_DatetimeDTypeMetaData *)dtype->c_metadata)->meta); /* Copy the metadata */ *dt_data = *meta; - /* Allocate a capsule for it (this claims ownership of dt_data) */ - metacobj = NpyCapsule_FromVoidPtr((void *)dt_data, simple_capsule_dtor); - if (metacobj == NULL) { - Py_DECREF(dtype); - return NULL; - } - - /* Set the metadata object in the dictionary. */ - if (PyDict_SetItemString(dtype->metadata, NPY_METADATA_DTSTR, - metacobj) < 0) { - Py_DECREF(dtype); - Py_DECREF(metacobj); - return NULL; - } - Py_DECREF(metacobj); - return dtype; } @@ -763,58 +729,19 @@ create_datetime_dtype_with_unit(int type_num, NPY_DATETIMEUNIT unit) } /* - * This function returns the a new reference to the - * capsule with the datetime metadata. - */ -NPY_NO_EXPORT PyObject * -get_datetime_metacobj_from_dtype(PyArray_Descr *dtype) -{ - PyObject *metacobj; - - /* Check that the dtype has metadata */ - if (dtype->metadata == NULL) { - PyErr_SetString(PyExc_TypeError, - "Datetime type object is invalid, lacks metadata"); - return NULL; - } - - /* Check that the dtype has unit metadata */ - metacobj = PyDict_GetItemString(dtype->metadata, NPY_METADATA_DTSTR); - if (metacobj == NULL) { - PyErr_SetString(PyExc_TypeError, - "Datetime type object is invalid, lacks unit metadata"); - return NULL; - } - - Py_INCREF(metacobj); - return metacobj; -} - -/* * This function returns a pointer to the DateTimeMetaData * contained within the provided datetime dtype. */ NPY_NO_EXPORT PyArray_DatetimeMetaData * get_datetime_metadata_from_dtype(PyArray_Descr *dtype) { - PyObject *metacobj; - PyArray_DatetimeMetaData *meta = NULL; - - metacobj = get_datetime_metacobj_from_dtype(dtype); - if (metacobj == NULL) { - return NULL; - } - - /* Check that the dtype has an NpyCapsule for the metadata */ - meta = (PyArray_DatetimeMetaData *)NpyCapsule_AsVoidPtr(metacobj); - Py_DECREF(metacobj); - if (meta == NULL) { + if (!PyDataType_ISDATETIME(dtype)) { PyErr_SetString(PyExc_TypeError, - "Datetime type object is invalid, unit metadata is corrupt"); + "cannot get datetime metadata from non-datetime type"); return NULL; } - return meta; + return &(((PyArray_DatetimeDTypeMetaData *)dtype->c_metadata)->meta); } /* @@ -1741,57 +1668,6 @@ units_overflow: { } /* - * Computes the GCD of the two date-time metadata values. Raises - * an exception if there is no reasonable GCD, such as with - * years and days. - * - * Returns a capsule with the GCD metadata. - */ -NPY_NO_EXPORT PyObject * -compute_datetime_metadata_greatest_common_divisor_capsule( - PyArray_Descr *type1, - PyArray_Descr *type2, - int strict_with_nonlinear_units1, - int strict_with_nonlinear_units2) -{ - PyArray_DatetimeMetaData *meta1, *meta2, *dt_data; - - if ((type1->type_num != NPY_DATETIME && - type1->type_num != NPY_TIMEDELTA) || - (type2->type_num != NPY_DATETIME && - type2->type_num != NPY_TIMEDELTA)) { - PyErr_SetString(PyExc_TypeError, - "Require datetime types for metadata " - "greatest common divisor operation"); - return NULL; - } - - meta1 = get_datetime_metadata_from_dtype(type1); - if (meta1 == NULL) { - return NULL; - } - meta2 = get_datetime_metadata_from_dtype(type2); - if (meta2 == NULL) { - return NULL; - } - - /* Create and return the metadata capsule */ - dt_data = PyArray_malloc(sizeof(PyArray_DatetimeMetaData)); - if (dt_data == NULL) { - return PyErr_NoMemory(); - } - - if (compute_datetime_metadata_greatest_common_divisor(meta1, meta2, - dt_data, strict_with_nonlinear_units1, - strict_with_nonlinear_units2) < 0) { - PyArray_free(dt_data); - return NULL; - } - - return NpyCapsule_FromVoidPtr((void *)dt_data, simple_capsule_dtor); -} - -/* * Both type1 and type2 must be either NPY_DATETIME or NPY_TIMEDELTA. * Applies the type promotion rules between the two types, returning * the promoted type. @@ -1800,7 +1676,6 @@ NPY_NO_EXPORT PyArray_Descr * datetime_type_promotion(PyArray_Descr *type1, PyArray_Descr *type2) { int type_num1, type_num2; - PyObject *gcdmeta; PyArray_Descr *dtype; int is_datetime; @@ -1809,50 +1684,28 @@ datetime_type_promotion(PyArray_Descr *type1, PyArray_Descr *type2) is_datetime = (type_num1 == NPY_DATETIME || type_num2 == NPY_DATETIME); - /* - * Get the metadata GCD, being strict about nonlinear units for - * timedelta and relaxed for datetime. - */ - gcdmeta = compute_datetime_metadata_greatest_common_divisor_capsule( - type1, type2, - type_num1 == NPY_TIMEDELTA, - type_num2 == NPY_TIMEDELTA); - if (gcdmeta == NULL) { - return NULL; - } - /* Create a DATETIME or TIMEDELTA dtype */ dtype = PyArray_DescrNewFromType(is_datetime ? NPY_DATETIME : NPY_TIMEDELTA); if (dtype == NULL) { - Py_DECREF(gcdmeta); return NULL; } - /* Replace the metadata dictionary */ - Py_XDECREF(dtype->metadata); - dtype->metadata = PyDict_New(); - if (dtype->metadata == NULL) { - Py_DECREF(dtype); - Py_DECREF(gcdmeta); - return NULL; - } - - /* Set the metadata object in the dictionary. */ - if (PyDict_SetItemString(dtype->metadata, NPY_METADATA_DTSTR, - gcdmeta) < 0) { + /* + * Get the metadata GCD, being strict about nonlinear units for + * timedelta and relaxed for datetime. + */ + if (compute_datetime_metadata_greatest_common_divisor( + get_datetime_metadata_from_dtype(type1), + get_datetime_metadata_from_dtype(type2), + get_datetime_metadata_from_dtype(dtype), + type_num1 == NPY_TIMEDELTA, + type_num2 == NPY_TIMEDELTA) < 0) { Py_DECREF(dtype); - Py_DECREF(gcdmeta); return NULL; } - Py_DECREF(gcdmeta); return dtype; - - - PyErr_SetString(PyExc_RuntimeError, - "Called datetime_type_promotion on non-datetype type"); - return NULL; } /* @@ -2021,25 +1874,6 @@ convert_datetime_metadata_tuple_to_datetime_metadata(PyObject *tuple, } /* - * Converts a metadata tuple into a datetime metadata capsule. - */ -NPY_NO_EXPORT PyObject * -convert_datetime_metadata_tuple_to_metacobj(PyObject *tuple) -{ - PyArray_DatetimeMetaData *dt_data; - - dt_data = PyArray_malloc(sizeof(PyArray_DatetimeMetaData)); - - if (convert_datetime_metadata_tuple_to_datetime_metadata( - tuple, dt_data) < 0) { - PyArray_free(dt_data); - return NULL; - } - - return NpyCapsule_FromVoidPtr((void *)dt_data, simple_capsule_dtor); -} - -/* * Converts an input object into datetime metadata. The input * may be either a string or a tuple. * @@ -2485,8 +2319,8 @@ get_tzoffset_from_pytzinfo(PyObject *timezone_obj, npy_datetimestruct *dts) Py_DECREF(loc_dt); /* Calculate the tzoffset as the difference between the datetimes */ - return get_datetimestruct_minutes(&loc_dts) - - get_datetimestruct_minutes(dts); + return (int)(get_datetimestruct_minutes(&loc_dts) - + get_datetimestruct_minutes(dts)); } /* diff --git a/numpy/core/src/multiarray/datetime_strings.c b/numpy/core/src/multiarray/datetime_strings.c index e9e344ae2..cfeabc783 100644 --- a/numpy/core/src/multiarray/datetime_strings.c +++ b/numpy/core/src/multiarray/datetime_strings.c @@ -328,7 +328,7 @@ convert_datetimestruct_local_to_utc(npy_datetimestruct *out_dts_utc, * Returns 0 on success, -1 on failure. */ NPY_NO_EXPORT int -parse_iso_8601_datetime(char *str, int len, +parse_iso_8601_datetime(char *str, Py_ssize_t len, NPY_DATETIMEUNIT unit, NPY_CASTING casting, npy_datetimestruct *out, @@ -338,7 +338,8 @@ parse_iso_8601_datetime(char *str, int len, { int year_leap = 0; int i, numdigits; - char *substr, sublen; + char *substr; + Py_ssize_t sublen; NPY_DATETIMEUNIT bestunit; /* Initialize the output to all zeros */ diff --git a/numpy/core/src/multiarray/datetime_strings.h b/numpy/core/src/multiarray/datetime_strings.h index f27856b89..4280f6de4 100644 --- a/numpy/core/src/multiarray/datetime_strings.h +++ b/numpy/core/src/multiarray/datetime_strings.h @@ -37,7 +37,7 @@ * Returns 0 on success, -1 on failure. */ NPY_NO_EXPORT int -parse_iso_8601_datetime(char *str, int len, +parse_iso_8601_datetime(char *str, Py_ssize_t len, NPY_DATETIMEUNIT unit, NPY_CASTING casting, npy_datetimestruct *out, diff --git a/numpy/core/src/multiarray/descriptor.c b/numpy/core/src/multiarray/descriptor.c index 724c2eb07..2095ef2ab 100644 --- a/numpy/core/src/multiarray/descriptor.c +++ b/numpy/core/src/multiarray/descriptor.c @@ -34,38 +34,36 @@ static PyObject *typeDict = NULL; /* Must be explicitly loaded */ static PyArray_Descr * _use_inherit(PyArray_Descr *type, PyObject *newobj, int *errflag); -NPY_NO_EXPORT PyArray_Descr * -_arraydescr_fromobj(PyObject *obj) +/* + * Creates a dtype object from ctypes inputs. + * + * Returns a new reference to a dtype object, or NULL + * if this is not possible. When it returns NULL, it does + * not set a Python exception. + */ +static PyArray_Descr * +_arraydescr_fromctypes(PyObject *obj) { PyObject *dtypedescr; - PyArray_Descr *new; + PyArray_Descr *newdescr; int ret; - dtypedescr = PyObject_GetAttrString(obj, "dtype"); - PyErr_Clear(); - if (dtypedescr) { - ret = PyArray_DescrConverter(dtypedescr, &new); - Py_DECREF(dtypedescr); - if (ret == NPY_SUCCEED) { - return new; - } - PyErr_Clear(); - } /* Understand basic ctypes */ dtypedescr = PyObject_GetAttrString(obj, "_type_"); PyErr_Clear(); if (dtypedescr) { - ret = PyArray_DescrConverter(dtypedescr, &new); + ret = PyArray_DescrConverter(dtypedescr, &newdescr); Py_DECREF(dtypedescr); if (ret == NPY_SUCCEED) { PyObject *length; + /* Check for ctypes arrays */ length = PyObject_GetAttrString(obj, "_length_"); PyErr_Clear(); if (length) { /* derived type */ PyObject *newtup; PyArray_Descr *derived; - newtup = Py_BuildValue("NO", new, length); + newtup = Py_BuildValue("NO", newdescr, length); ret = PyArray_DescrConverter(newtup, &derived); Py_DECREF(newtup); if (ret == NPY_SUCCEED) { @@ -74,7 +72,7 @@ _arraydescr_fromobj(PyObject *obj) PyErr_Clear(); return NULL; } - return new; + return newdescr; } PyErr_Clear(); return NULL; @@ -85,16 +83,53 @@ _arraydescr_fromobj(PyObject *obj) dtypedescr = PyObject_GetAttrString(obj, "_fields_"); PyErr_Clear(); if (dtypedescr) { - ret = PyArray_DescrAlignConverter(dtypedescr, &new); + ret = PyArray_DescrAlignConverter(dtypedescr, &newdescr); Py_DECREF(dtypedescr); if (ret == NPY_SUCCEED) { - return new; + return newdescr; } PyErr_Clear(); } + return NULL; } +/* + * This function creates a dtype object when: + * - The object has a "dtype" attribute, and it can be converted + * to a dtype object. + * - The object is a ctypes type object, including array + * and structure types. + * + * Returns a new reference to a dtype object, or NULL + * if this is not possible. When it returns NULL, it does + * not set a Python exception. + */ +NPY_NO_EXPORT PyArray_Descr * +_arraydescr_fromobj(PyObject *obj) +{ + PyObject *dtypedescr; + PyArray_Descr *newdescr = NULL; + int ret; + + /* For arbitrary objects that have a "dtype" attribute */ + dtypedescr = PyObject_GetAttrString(obj, "dtype"); + PyErr_Clear(); + if (dtypedescr != NULL) { + ret = PyArray_DescrConverter(dtypedescr, &newdescr); + Py_DECREF(dtypedescr); + if (ret == NPY_SUCCEED) { + return newdescr; + } + PyErr_Clear(); + } + return _arraydescr_fromctypes(obj); +} + +/* + * Sets the global typeDict object, which is a dictionary mapping + * dtype names to numpy scalar types. + */ NPY_NO_EXPORT PyObject * array_set_typeDict(PyObject *NPY_UNUSED(ignored), PyObject *args) { @@ -140,7 +175,10 @@ _check_for_commastring(char *type, Py_ssize_t len) && type[2] == ')'))) { return 1; } - /* Check for presence of commas outside square [] brackets */ + /* + * Check for presence of commas outside square [] brackets. This + * allows commas inside of [], for parameterized dtypes to use. + */ sqbracket = 0; for (i = 1; i < len; i++) { switch (type[i]) { @@ -310,9 +348,9 @@ _convert_from_tuple(PyObject *obj) * shape parameter). * * field-name can be a string or a 2-tuple - * data-type can now be a list, string, or 2-tuple (string, metadata dictionary)) + * data-type can now be a list, string, or 2-tuple + * (string, metadata dictionary) */ - static PyArray_Descr * _convert_from_array_descr(PyObject *obj, int align) { @@ -834,6 +872,9 @@ _use_fields_dict(PyObject *obj, int align) return res; } +/* + * Creates a struct dtype object from a Python dictionary. + */ static PyArray_Descr * _convert_from_dict(PyObject *obj, int align) { @@ -1443,7 +1484,7 @@ error: * If a mistake is made in reference counting, deallocation on these * builtins will be attempted leading to problems. * - * This let's us deal with all PyArray_Descr objects using reference + * This lets us deal with all PyArray_Descr objects using reference * counting (regardless of whether they are statically or dynamically * allocated). */ @@ -1454,31 +1495,47 @@ error: NPY_NO_EXPORT PyArray_Descr * PyArray_DescrNew(PyArray_Descr *base) { - PyArray_Descr *new = PyObject_New(PyArray_Descr, &PyArrayDescr_Type); + PyArray_Descr *newdescr = PyObject_New(PyArray_Descr, &PyArrayDescr_Type); - if (new == NULL) { + if (newdescr == NULL) { return NULL; } /* Don't copy PyObject_HEAD part */ - memcpy((char *)new + sizeof(PyObject), + memcpy((char *)newdescr + sizeof(PyObject), (char *)base + sizeof(PyObject), sizeof(PyArray_Descr) - sizeof(PyObject)); - if (new->fields == Py_None) { - new->fields = NULL; + /* + * The c_metadata has a by-value ownership model, need to clone it + * (basically a deep copy, but the auxdata clone function has some + * flexibility still) so the new PyArray_Descr object owns + * a copy of the data. Having both 'base' and 'newdescr' point to + * the same auxdata pointer would cause a double-free of memory. + */ + if (base->c_metadata != NULL) { + newdescr->c_metadata = NPY_AUXDATA_CLONE(base->c_metadata); + if (newdescr->c_metadata == NULL) { + PyErr_NoMemory(); + Py_DECREF(newdescr); + return NULL; + } + } + + if (newdescr->fields == Py_None) { + newdescr->fields = NULL; } - Py_XINCREF(new->fields); - Py_XINCREF(new->names); - if (new->subarray) { - new->subarray = PyArray_malloc(sizeof(PyArray_ArrayDescr)); - memcpy(new->subarray, base->subarray, sizeof(PyArray_ArrayDescr)); - Py_INCREF(new->subarray->shape); - Py_INCREF(new->subarray->base); + Py_XINCREF(newdescr->fields); + Py_XINCREF(newdescr->names); + if (newdescr->subarray) { + newdescr->subarray = PyArray_malloc(sizeof(PyArray_ArrayDescr)); + memcpy(newdescr->subarray, base->subarray, sizeof(PyArray_ArrayDescr)); + Py_INCREF(newdescr->subarray->shape); + Py_INCREF(newdescr->subarray->base); } - Py_XINCREF(new->typeobj); - Py_XINCREF(new->metadata); + Py_XINCREF(newdescr->typeobj); + Py_XINCREF(newdescr->metadata); - return new; + return newdescr; } /* @@ -1505,6 +1562,8 @@ arraydescr_dealloc(PyArray_Descr *self) PyArray_free(self->subarray); } Py_XDECREF(self->metadata); + NPY_AUXDATA_FREE(self->c_metadata); + self->c_metadata = NULL; Py_TYPE(self)->tp_free((PyObject *)self); } @@ -1940,23 +1999,6 @@ static PyGetSetDef arraydescr_getsets[] = { {NULL, NULL, NULL, NULL, NULL}, }; -static int -_invalid_metadata_check(PyObject *metadata) -{ - PyObject *res; - - /* borrowed reference */ - res = PyDict_GetItemString(metadata, NPY_METADATA_DTSTR); - if (res == NULL) { - return 0; - } - else { - PyErr_SetString(PyExc_ValueError, - "cannot set " NPY_METADATA_DTSTR "in dtype metadata"); - return 1; - } -} - static PyObject * arraydescr_new(PyTypeObject *NPY_UNUSED(subtype), PyObject *args, PyObject *kwds) @@ -1977,10 +2019,6 @@ arraydescr_new(PyTypeObject *NPY_UNUSED(subtype), return NULL; } - if ((metadata != NULL) && (_invalid_metadata_check(metadata))) { - return NULL; - } - if (align) { if (!PyArray_DescrAlignConverter(odescr, &conv)) { return NULL; @@ -2056,14 +2094,13 @@ _get_pickleabletype_from_datetime_metadata(PyArray_Descr *dtype) return NULL; } - /* Make a cleaned copy of the metadata dictionary */ - newdict = PyDict_Copy(dtype->metadata); - if (newdict == NULL) { - Py_DECREF(ret); - return NULL; + /* Store the metadata dictionary */ + if (dtype->metadata != NULL) { + Py_INCREF(dtype->metadata); + PyTuple_SET_ITEM(ret, 0, dtype->metadata); + } else { + PyTuple_SET_ITEM(ret, 0, PyDict_New()); } - PyDict_DelItemString(newdict, NPY_METADATA_DTSTR); - PyTuple_SET_ITEM(ret, 0, newdict); /* Convert the datetime metadata into a tuple */ meta = get_datetime_metadata_from_dtype(dtype); @@ -2071,11 +2108,21 @@ _get_pickleabletype_from_datetime_metadata(PyArray_Descr *dtype) Py_DECREF(ret); return NULL; } - dt_tuple = convert_datetime_metadata_to_tuple(meta); + /* Use a 4-tuple that numpy 1.6 knows how to unpickle */ + dt_tuple = PyTuple_New(4); if (dt_tuple == NULL) { Py_DECREF(ret); return NULL; } + PyTuple_SET_ITEM(dt_tuple, 0, + PyBytes_FromString(_datetime_strings[meta->base])); + PyTuple_SET_ITEM(dt_tuple, 1, + PyInt_FromLong(meta->num)); + PyTuple_SET_ITEM(dt_tuple, 2, + PyInt_FromLong(1)); + PyTuple_SET_ITEM(dt_tuple, 3, + PyInt_FromLong(1)); + PyTuple_SET_ITEM(ret, 1, dt_tuple); return ret; @@ -2144,28 +2191,27 @@ arraydescr_reduce(PyArray_Descr *self, PyObject *NPY_UNUSED(args)) endian = '>'; } } - if (self->metadata) { + if (PyDataType_ISDATETIME(self)) { + PyObject *newobj; state = PyTuple_New(9); PyTuple_SET_ITEM(state, 0, PyInt_FromLong(version)); - if (PyDataType_ISDATETIME(self)) { - PyObject *newobj; - /* Handle CObject in NPY_METADATA_DTSTR key separately */ - /* - * newobj is a tuple of cleaned metadata dictionary - * and tuple of date_time info (str, num) - */ - newobj = _get_pickleabletype_from_datetime_metadata(self); - if (newobj == NULL) { - Py_DECREF(state); - Py_DECREF(ret); - return NULL; - } - PyTuple_SET_ITEM(state, 8, newobj); - } - else { - Py_INCREF(self->metadata); - PyTuple_SET_ITEM(state, 8, self->metadata); + /* + * newobj is a tuple of the Python metadata dictionary + * and tuple of date_time info (str, num) + */ + newobj = _get_pickleabletype_from_datetime_metadata(self); + if (newobj == NULL) { + Py_DECREF(state); + Py_DECREF(ret); + return NULL; } + PyTuple_SET_ITEM(state, 8, newobj); + } + else if (self->metadata) { + state = PyTuple_New(9); + PyTuple_SET_ITEM(state, 0, PyInt_FromLong(version)); + Py_INCREF(self->metadata); + PyTuple_SET_ITEM(state, 8, self->metadata); } else { /* Use version 3 pickle format */ state = PyTuple_New(8); @@ -2477,29 +2523,30 @@ arraydescr_setstate(PyArray_Descr *self, PyObject *args) self->flags = _descr_find_object(self); } + /* + * We have a borrowed reference to metadata so no need + * to alter reference count when throwing away Py_None. + */ + if (metadata == Py_None) { + metadata = NULL; + } + Py_XDECREF(self->metadata); - if (PyDataType_ISDATETIME(self) - && (metadata != Py_None) - && (metadata != NULL)) { - PyObject *cobj; + if (PyDataType_ISDATETIME(self) && (metadata != NULL)) { + PyArray_DatetimeMetaData *dt_data; + + /* The Python metadata */ self->metadata = PyTuple_GET_ITEM(metadata, 0); - Py_INCREF(self->metadata); - cobj = convert_datetime_metadata_tuple_to_metacobj( - PyTuple_GET_ITEM(metadata, 1)); - if (cobj == NULL) { + + /* The datetime metadata */ + dt_data = &(((PyArray_DatetimeDTypeMetaData *)self->c_metadata)->meta); + if (convert_datetime_metadata_tuple_to_datetime_metadata( + PyTuple_GET_ITEM(metadata, 1), + dt_data) < 0) { return NULL; } - PyDict_SetItemString(self->metadata, NPY_METADATA_DTSTR, cobj); - Py_DECREF(cobj); } else { - /* - * We have a borrowed reference to metadata so no need - * to alter reference count - */ - if (metadata == Py_None) { - metadata = NULL; - } self->metadata = metadata; Py_XINCREF(metadata); } diff --git a/numpy/core/src/multiarray/multiarraymodule.c b/numpy/core/src/multiarray/multiarraymodule.c index 6cbdf2eb9..7a657d8b9 100644 --- a/numpy/core/src/multiarray/multiarraymodule.c +++ b/numpy/core/src/multiarray/multiarraymodule.c @@ -4132,10 +4132,6 @@ PyMODINIT_FUNC initmultiarray(void) { PyDict_SetItemString(d, "__version__", s); Py_DECREF(s); - s = PyUString_InternFromString(NPY_METADATA_DTSTR); - PyDict_SetItemString(d, "METADATA_DTSTR", s); - Py_DECREF(s); - /* FIXME * There is no error handling here */ diff --git a/numpy/core/src/multiarray/scalarapi.c b/numpy/core/src/multiarray/scalarapi.c index f3b2eb16e..8b61c797d 100644 --- a/numpy/core/src/multiarray/scalarapi.c +++ b/numpy/core/src/multiarray/scalarapi.c @@ -532,38 +532,19 @@ PyArray_DescrFromScalar(PyObject *sc) } if (PyArray_IsScalar(sc, Datetime) || PyArray_IsScalar(sc, Timedelta)) { - PyObject *cobj; PyArray_DatetimeMetaData *dt_data; - dt_data = PyArray_malloc(sizeof(PyArray_DatetimeMetaData)); if (PyArray_IsScalar(sc, Datetime)) { descr = PyArray_DescrNewFromType(NPY_DATETIME); - memcpy(dt_data, &((PyDatetimeScalarObject *)sc)->obmeta, - sizeof(PyArray_DatetimeMetaData)); } else { /* Timedelta */ descr = PyArray_DescrNewFromType(NPY_TIMEDELTA); - memcpy(dt_data, &((PyTimedeltaScalarObject *)sc)->obmeta, - sizeof(PyArray_DatetimeMetaData)); - } - cobj = NpyCapsule_FromVoidPtr((void *)dt_data, simple_capsule_dtor); - - /* Add correct meta-data to the data-type */ - if (descr == NULL) { - Py_DECREF(cobj); - return NULL; - } - Py_XDECREF(descr->metadata); - if ((descr->metadata = PyDict_New()) == NULL) { - Py_DECREF(descr); - Py_DECREF(cobj); - return NULL; } + dt_data = &(((PyArray_DatetimeDTypeMetaData *)descr->c_metadata)->meta); + memcpy(dt_data, &((PyDatetimeScalarObject *)sc)->obmeta, + sizeof(PyArray_DatetimeMetaData)); - /* Assume this sets a new reference to cobj */ - PyDict_SetItemString(descr->metadata, NPY_METADATA_DTSTR, cobj); - Py_DECREF(cobj); return descr; } @@ -676,14 +657,9 @@ PyArray_Scalar(void *data, PyArray_Descr *descr, PyObject *base) * We need to copy the resolution information over to the scalar * Get the void * from the metadata dictionary */ - PyObject *cobj; PyArray_DatetimeMetaData *dt_data; - cobj = PyDict_GetItemString(descr->metadata, NPY_METADATA_DTSTR); -/* FIXME - * There is no error handling here. - */ - dt_data = NpyCapsule_AsVoidPtr(cobj); + dt_data = &(((PyArray_DatetimeDTypeMetaData *)descr->c_metadata)->meta); memcpy(&(((PyDatetimeScalarObject *)obj)->obmeta), dt_data, sizeof(PyArray_DatetimeMetaData)); } diff --git a/numpy/core/src/umath/ufunc_type_resolution.c b/numpy/core/src/umath/ufunc_type_resolution.c index 83703328a..3c697f1ee 100644 --- a/numpy/core/src/umath/ufunc_type_resolution.c +++ b/numpy/core/src/umath/ufunc_type_resolution.c @@ -508,38 +508,6 @@ PyUFunc_AbsoluteTypeResolver(PyUFuncObject *ufunc, } } - -/* - * This function returns the a new reference to the - * capsule with the datetime metadata. - * - * NOTE: This function is copied from datetime.c in multiarray, - * because umath and multiarray are not linked together. - */ -static PyObject * -get_datetime_metacobj_from_dtype(PyArray_Descr *dtype) -{ - PyObject *metacobj; - - /* Check that the dtype has metadata */ - if (dtype->metadata == NULL) { - PyErr_SetString(PyExc_TypeError, - "Datetime type object is invalid, lacks metadata"); - return NULL; - } - - /* Check that the dtype has unit metadata */ - metacobj = PyDict_GetItemString(dtype->metadata, NPY_METADATA_DTSTR); - if (metacobj == NULL) { - PyErr_SetString(PyExc_TypeError, - "Datetime type object is invalid, lacks unit metadata"); - return NULL; - } - - Py_INCREF(metacobj); - return metacobj; -} - /* * Creates a new NPY_TIMEDELTA dtype, copying the datetime metadata * from the given dtype. @@ -551,31 +519,20 @@ static PyArray_Descr * timedelta_dtype_with_copied_meta(PyArray_Descr *dtype) { PyArray_Descr *ret; - PyObject *metacobj; + PyArray_DatetimeMetaData *dst, *src; + PyArray_DatetimeDTypeMetaData *dst_dtmd, *src_dtmd; ret = PyArray_DescrNewFromType(NPY_TIMEDELTA); if (ret == NULL) { return NULL; } - Py_XDECREF(ret->metadata); - ret->metadata = PyDict_New(); - if (ret->metadata == NULL) { - Py_DECREF(ret); - return NULL; - } - metacobj = get_datetime_metacobj_from_dtype(dtype); - if (metacobj == NULL) { - Py_DECREF(ret); - return NULL; - } + src_dtmd = ((PyArray_DatetimeDTypeMetaData *)dtype->c_metadata); + dst_dtmd = ((PyArray_DatetimeDTypeMetaData *)ret->c_metadata); + src = &(src_dtmd->meta); + dst = &(dst_dtmd->meta); - if (PyDict_SetItemString(ret->metadata, NPY_METADATA_DTSTR, - metacobj) < 0) { - Py_DECREF(metacobj); - Py_DECREF(ret); - return NULL; - } + *dst = *src; return ret; } diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py index f062645d2..cb62182cd 100644 --- a/numpy/core/tests/test_datetime.py +++ b/numpy/core/tests/test_datetime.py @@ -529,9 +529,23 @@ class TestDateTime(TestCase): def test_pickle(self): # Check that pickle roundtripping works dt = np.dtype('M8[7D]') - assert_equal(dt, pickle.loads(pickle.dumps(dt))) + assert_equal(pickle.loads(pickle.dumps(dt)), dt) dt = np.dtype('M8[W]') - assert_equal(dt, pickle.loads(pickle.dumps(dt))) + assert_equal(pickle.loads(pickle.dumps(dt)), dt) + + # Check that loading pickles from 1.6 works + pkl = "cnumpy\ndtype\np0\n(S'M8'\np1\nI0\nI1\ntp2\nRp3\n" + \ + "(I4\nS'<'\np4\nNNNI-1\nI-1\nI0\n((dp5\n(S'D'\np6\n" + \ + "I7\nI1\nI1\ntp7\ntp8\ntp9\nb." + assert_equal(pickle.loads(asbytes(pkl)), np.dtype('<M8[7D]')) + pkl = "cnumpy\ndtype\np0\n(S'M8'\np1\nI0\nI1\ntp2\nRp3\n" + \ + "(I4\nS'<'\np4\nNNNI-1\nI-1\nI0\n((dp5\n(S'W'\np6\n" + \ + "I1\nI1\nI1\ntp7\ntp8\ntp9\nb." + assert_equal(pickle.loads(asbytes(pkl)), np.dtype('<M8[W]')) + pkl = "cnumpy\ndtype\np0\n(S'M8'\np1\nI0\nI1\ntp2\nRp3\n" + \ + "(I4\nS'>'\np4\nNNNI-1\nI-1\nI0\n((dp5\n(S'us'\np6\n" + \ + "I1\nI1\nI1\ntp7\ntp8\ntp9\nb." + assert_equal(pickle.loads(asbytes(pkl)), np.dtype('>M8[us]')) def test_dtype_promotion(self): # datetime <op> datetime computes the metadata gcd diff --git a/numpy/lib/type_check.py b/numpy/lib/type_check.py index edea02b62..c116c7e4a 100644 --- a/numpy/lib/type_check.py +++ b/numpy/lib/type_check.py @@ -8,7 +8,6 @@ __all__ = ['iscomplexobj','isrealobj','imag','iscomplex', import numpy.core.numeric as _nx from numpy.core.numeric import asarray, asanyarray, array, isnan, \ obj2sctype, zeros -from numpy.core.multiarray import METADATA_DTSTR from ufunclike import isneginf, isposinf _typecodes_by_elsize = 'GDFgdfQqLlIiHhBb?' |