ENH: datetime: Eliminate use of npy_timedeltastruct, an unnecessary abstraction

5 files changed, 266 insertions, 188 deletions

diff --git a/numpy/core/include/numpy/ndarraytypes.h b/numpy/core/include/numpy/ndarraytypes.h
index 3e6af3c08..13a4fb7a5 100644
--- a/numpy/core/include/numpy/ndarraytypes.h
+++ b/numpy/core/include/numpy/ndarraytypes.h
@@ -697,12 +697,14 @@ typedef struct {
         npy_int32 event;
 } npy_datetimestruct;
 
+/* TO BE REMOVED - NOT USED INTERNALLY. */
 typedef struct {
         npy_int64 day;
         npy_int32 sec, us, ps, as;
         npy_int32 event;
 } npy_timedeltastruct;
 
+/* TO BE REMOVED - NOT USED INTERNALLY. */
 #if PY_VERSION_HEX >= 0x03000000
 #define PyDataType_GetDatetimeMetaData(descr)                                 \
     ((descr->metadata == NULL) ? NULL :                                       \
diff --git a/numpy/core/src/multiarray/_datetime.h b/numpy/core/src/multiarray/_datetime.h
index 4c7e4457b..820b4f7e9 100644
--- a/numpy/core/src/multiarray/_datetime.h
+++ b/numpy/core/src/multiarray/_datetime.h
@@ -4,20 +4,6 @@
 NPY_NO_EXPORT void
 numpy_pydatetime_import();
 
-NPY_NO_EXPORT void
-PyArray_DatetimeToDatetimeStruct(npy_datetime val, NPY_DATETIMEUNIT fr,
-                                 npy_datetimestruct *result);
-
-NPY_NO_EXPORT void
-PyArray_TimedeltaToTimedeltaStruct(npy_timedelta val, NPY_DATETIMEUNIT fr,
-                                 npy_timedeltastruct *result);
-
-NPY_NO_EXPORT npy_datetime
-PyArray_DatetimeStructToDatetime(NPY_DATETIMEUNIT fr, npy_datetimestruct *d);
-
-NPY_NO_EXPORT npy_datetime
-PyArray_TimedeltaStructToTimedelta(NPY_DATETIMEUNIT fr, npy_timedeltastruct *d);
-
 /*
  * This function returns the a new reference to the
  * capsule with the datetime metadata.
@@ -242,6 +228,16 @@ NPY_NO_EXPORT PyObject *
 convert_datetime_to_pyobject(npy_datetime dt, PyArray_DatetimeMetaData *meta);
 
 /*
+ * Converts a timedelta into a PyObject *.
+ *
+ * Not-a-time is returned as the string "NaT".
+ * For microseconds or coarser, returns a datetime.timedelta.
+ * For units finer than microseconds, returns an integer.
+ */
+NPY_NO_EXPORT PyObject *
+convert_timedelta_to_pyobject(npy_timedelta td, PyArray_DatetimeMetaData *meta);
+
+/*
  * Converts a datetime based on the given metadata into a datetimestruct
  */
 NPY_NO_EXPORT int
diff --git a/numpy/core/src/multiarray/arraytypes.c.src b/numpy/core/src/multiarray/arraytypes.c.src
index e4fe2c000..3059c4109 100644
--- a/numpy/core/src/multiarray/arraytypes.c.src
+++ b/numpy/core/src/multiarray/arraytypes.c.src
@@ -106,8 +106,6 @@ MyPyLong_AsUnsigned@Type@ (PyObject *obj)
  */
 
 
-static char * _SEQUENCE_MESSAGE = "error setting an array element with a sequence";
-
 /**begin repeat
  *
  * #TYPE = BOOL, BYTE, UBYTE, SHORT, USHORT, INT, LONG, UINT, ULONG,
@@ -764,141 +762,6 @@ fail:
     return -1;
 }
 
-/*
- * Acknowledgement:  Example code contributed by Marty Fuhr sponsored by
- * Google Summer of Code 2009 was used to integrate and adapt the mxDateTime
- * parser
- */
-
-/* #include "datetime.c" --- now included in multiarray_onefile */
-
-
-/* DateTime Objects in Python only keep microsecond resolution.
- *
- * When converting from datetime objects with an event component return a
- * tuple: * (baseunit, number of event)  where baseunit follows is a datetime
- * type and number of events is a Python integer
- */
-
-
-/*
- * We also can lose precision and range here.  Ignored.
- * Don't use this function if you care.
- */
-
-NPY_NO_EXPORT PyObject *
-PyTimeDelta_FromNormalized(npy_timedelta val, NPY_DATETIMEUNIT base)
-{
-    npy_timedeltastruct td;
-
-    PyDateTime_IMPORT;
-    PyArray_TimedeltaToTimedeltaStruct(val, base, &td);
-
-    /* We discard td.ps and td.as */
-    return PyDelta_FromDSU(td.day, td.sec, td.us);
-}
-
-NPY_NO_EXPORT PyObject *
-PyTimeDelta_FromInt64(timedelta val, PyArray_Descr *descr)
-{
-    PyArray_DatetimeMetaData *meta;
-    meta = PyDataType_GetDatetimeMetaData(descr);
-    if (meta == NULL) {
-        PyErr_SetString(PyExc_RuntimeError,
-                "metadata not set for descriptor");
-        return NULL;
-    }
-
-    if (meta->events > 1) {
-        int events, rem, div;
-        PyObject *obj;
-
-        obj = PyTuple_New(2);
-        events = meta->events;
-        div = val/events;
-        rem = val % events;
-        PyTuple_SET_ITEM(obj, 1, PyInt_FromLong(rem));
-        /* This resets meta->events for recursive call */
-        meta->events = 1;
-        PyTuple_SET_ITEM(obj, 0, PyTimeDelta_FromInt64(div, descr));
-        meta->events = events;
-        if (PyErr_Occurred()) {
-            Py_DECREF(obj);
-            return NULL;
-        }
-        return obj;
-    }
-
-    /* FIXME? : Check for Overflow */
-    return PyTimeDelta_FromNormalized(val*meta->num, meta->base);
-}
-
-NPY_NO_EXPORT npy_timedelta
-PyTimeDelta_AsNormalized(PyObject *obj, NPY_DATETIMEUNIT base)
-{
-    npy_timedeltastruct td;
-
-    PyDateTime_IMPORT;
-
-    if (!PyDelta_Check(obj)) {
-        PyErr_SetString(PyExc_ValueError,
-                "Must be a datetime.timedelta object");
-        return -1;
-    }
-
-    td.day = ((PyDateTime_Delta *)obj)->days;
-    td.sec = ((PyDateTime_Delta *)obj)->seconds;
-    td.us  = ((PyDateTime_Delta *)obj)->microseconds;
-    td.ps  = 0;
-    td.as  = 0;
-
-    return PyArray_TimedeltaStructToTimedelta(base, &td);
-}
-
-NPY_NO_EXPORT timedelta
-PyTimeDelta_AsInt64(PyObject *obj, PyArray_Descr *descr)
-{
-    PyArray_DatetimeMetaData *meta;
-    npy_timedelta res;
-
-    meta = PyDataType_GetDatetimeMetaData(descr);
-    if (meta == NULL) {
-        PyErr_SetString(PyExc_RuntimeError,
-                "metadata not set for descriptor");
-        return -1;
-    }
-
-    if (meta->events > 1) {
-        timedelta tmp;
-        int events;
-
-        if (!PyTuple_Check(obj) || PyTuple_GET_SIZE(obj) != 2) {
-            PyErr_SetString(PyExc_ValueError,
-                    "need a 2-tuple on setting if events > 1");
-            return -1;
-        }
-        /* Alter the dictionary and call again (not thread safe) */
-        events = meta->events;
-        meta->events = 1;
-        tmp = PyTimeDelta_AsInt64(PyTuple_GET_ITEM(obj, 0), descr);
-        meta->events = events;
-        if (PyErr_Occurred()) {
-            return -1;
-        }
-        /* FIXME: Check for overflow */
-        tmp *= events;
-        tmp += MyPyLong_AsLongLong(PyTuple_GET_ITEM(obj, 1));
-        if (PyErr_Occurred()) {
-            return -1;
-        }
-        return tmp;
-    }
-
-    res = PyTimeDelta_AsNormalized(obj, meta->base);
-    return res / meta->num;
-}
-
-
 static PyObject *
 DATETIME_getitem(char *ip, PyArrayObject *ap) {
     npy_datetime dt;
@@ -911,7 +774,7 @@ DATETIME_getitem(char *ip, PyArrayObject *ap) {
     }
 
     if ((ap == NULL) || PyArray_ISBEHAVED_RO(ap)) {
-        dt = *((datetime *)ip);
+        dt = *((npy_datetime *)ip);
     }
     else {
         ap->descr->f->copyswap(&dt, ip, !PyArray_ISNOTSWAPPED(ap), ap);
@@ -923,16 +786,23 @@ DATETIME_getitem(char *ip, PyArrayObject *ap) {
 
 static PyObject *
 TIMEDELTA_getitem(char *ip, PyArrayObject *ap) {
-    timedelta t1;
+    npy_timedelta td;
+    PyArray_DatetimeMetaData *meta = NULL;
+
+    /* Get the datetime units metadata */
+    meta = get_datetime_metadata_from_dtype(PyArray_DESCR(ap));
+    if (meta == NULL) {
+        return NULL;
+    }
 
     if ((ap == NULL) || PyArray_ISBEHAVED_RO(ap)) {
-        t1 = *((timedelta *)ip);
-        return PyTimeDelta_FromInt64((timedelta)t1, ap->descr);
+        td = *((npy_timedelta *)ip);
     }
     else {
-        ap->descr->f->copyswap(&t1, ip, !PyArray_ISNOTSWAPPED(ap), ap);
-        return PyTimeDelta_FromInt64((timedelta)t1, ap->descr);
+        ap->descr->f->copyswap(&td, ip, !PyArray_ISNOTSWAPPED(ap), ap);
     }
+
+    return convert_timedelta_to_pyobject(td, meta);
 }
 
 static int
@@ -954,7 +824,7 @@ DATETIME_setitem(PyObject *op, char *ov, PyArrayObject *ap) {
 
     /* Copy the value into the output */
     if (ap == NULL || PyArray_ISBEHAVED(ap)) {
-        *((datetime *)ov)=temp;
+        *((npy_datetime *)ov)=temp;
     }
     else {
         ap->descr->f->copyswap(ov, &temp, !PyArray_ISNOTSWAPPED(ap), ap);
@@ -963,43 +833,31 @@ DATETIME_setitem(PyObject *op, char *ov, PyArrayObject *ap) {
     return 0;
 }
 
-/* FIXME: This needs to take
- *    1) Integers and Longs (anything that can be converted to an Int)
- *    2) Timedelta scalar objects (with resolution conversion)
- *    3) Python Timedelta objects
- *
- *    Plus a tuple for meta->events > 1
- */
-
 static int
 TIMEDELTA_setitem(PyObject *op, char *ov, PyArrayObject *ap) {
     /* ensure alignment */
-    timedelta temp;
+    npy_timedelta temp = 0;
+    PyArray_DatetimeMetaData *meta = NULL;
 
-    if (PyArray_IsScalar(op, Timedelta)) {
-        temp = ((PyTimedeltaScalarObject *)op)->obval;
-    }
-    else if (PyInt_Check(op)) {
-        temp = PyInt_AS_LONG(op);
-    }
-    else if (PyLong_Check(op)) {
-        temp = PyLong_AsLongLong(op);
-    }
-    else {
-        temp = PyTimeDelta_AsInt64(op, ap->descr);
+    /* Get the datetime units metadata */
+    meta = get_datetime_metadata_from_dtype(PyArray_DESCR(ap));
+    if (meta == NULL) {
+        return -1;
     }
-    if (PyErr_Occurred()) {
-        if (PySequence_Check(op)) {
-            PyErr_Clear();
-            PyErr_SetString(PyExc_ValueError, _SEQUENCE_MESSAGE);
-        }
+
+    /* Convert the object into a NumPy datetime */
+    if (convert_pyobject_to_timedelta(meta, op, &temp) < 0) {
         return -1;
     }
-    if (ap == NULL || PyArray_ISBEHAVED(ap))
-        *((timedelta *)ov)=temp;
+
+    /* Copy the value into the output */
+    if (ap == NULL || PyArray_ISBEHAVED(ap)) {
+        *((npy_timedelta *)ov)=temp;
+    }
     else {
         ap->descr->f->copyswap(ov, &temp, !PyArray_ISNOTSWAPPED(ap), ap);
     }
+
     return 0;
 }
 
diff --git a/numpy/core/src/multiarray/datetime.c b/numpy/core/src/multiarray/datetime.c
index 84297198a..7923bac66 100644
--- a/numpy/core/src/multiarray/datetime.c
+++ b/numpy/core/src/multiarray/datetime.c
@@ -440,6 +440,8 @@ convert_datetimestruct_to_datetime(PyArray_DatetimeMetaData *meta,
 
 /*NUMPY_API
  * Create a datetime value from a filled datetime struct and resolution unit.
+ *
+ * TO BE REMOVED - NOT USED INTERNALLY.
  */
 NPY_NO_EXPORT npy_datetime
 PyArray_DatetimeStructToDatetime(NPY_DATETIMEUNIT fr, npy_datetimestruct *d)
@@ -467,6 +469,8 @@ PyArray_DatetimeStructToDatetime(NPY_DATETIMEUNIT fr, npy_datetimestruct *d)
 
 /*NUMPY_API
  * Create a timdelta value from a filled timedelta struct and resolution unit.
+ *
+ * TO BE REMOVED - NOT USED INTERNALLY.
  */
 NPY_NO_EXPORT npy_datetime
 PyArray_TimedeltaStructToTimedelta(NPY_DATETIMEUNIT fr, npy_timedeltastruct *d)
@@ -847,6 +851,8 @@ convert_datetime_to_datetimestruct(PyArray_DatetimeMetaData *meta,
 
 /*NUMPY_API
  * Fill the datetime struct from the value and resolution unit.
+ *
+ * TO BE REMOVED - NOT USED INTERNALLY.
  */
 NPY_NO_EXPORT void
 PyArray_DatetimeToDatetimeStruct(npy_datetime val, NPY_DATETIMEUNIT fr,
@@ -875,6 +881,8 @@ PyArray_DatetimeToDatetimeStruct(npy_datetime val, NPY_DATETIMEUNIT fr,
 
 /*NUMPY_API
  * Fill the timedelta struct from the timedelta value and resolution unit.
+ *
+ * TO BE REMOVED - NOT USED INTERNALLY.
  */
 NPY_NO_EXPORT void
 PyArray_TimedeltaToTimedeltaStruct(npy_timedelta val, NPY_DATETIMEUNIT fr,
@@ -2392,9 +2400,15 @@ parse_iso_8601_date(char *str, int len, npy_datetimestruct *out)
                     tolower(str[1]) == 'o' &&
                     tolower(str[2]) == 'w') {
         time_t rawtime = 0;
+        PyArray_DatetimeMetaData meta;
         time(&rawtime);
-        PyArray_DatetimeToDatetimeStruct(rawtime, NPY_FR_s, out);
-        return 0;
+
+        /* Set up a dummy metadata for the conversion */
+        meta.base = NPY_FR_s;
+        meta.num = 1;
+        meta.events = 1;
+
+        return convert_datetime_to_datetimestruct(&meta, rawtime, out);
     }
 
     substr = str;
@@ -3161,7 +3175,63 @@ convert_pyobject_to_timedelta(PyArray_DatetimeMetaData *meta, PyObject *obj,
 
         return cast_timedelta_to_timedelta(obj_meta, meta, dt, out);
     }
-    /* TODO: Finish this function */
+    /* Convert from a Python timedelta object */
+    else if (PyObject_HasAttrString(obj, "days") &&
+                PyObject_HasAttrString(obj, "seconds") &&
+                PyObject_HasAttrString(obj, "microseconds")) {
+        PyObject *tmp;
+        PyArray_DatetimeMetaData us_meta;
+        npy_timedelta td;
+        npy_int64 days;
+        int seconds = 0, useconds = 0;
+
+        /* Get the days */
+        tmp = PyObject_GetAttrString(obj, "days");
+        if (tmp == NULL) {
+            return -1;
+        }
+        days = PyLong_AsLongLong(tmp);
+        if (days == -1 && PyErr_Occurred()) {
+            Py_DECREF(tmp);
+            return -1;
+        }
+        Py_DECREF(tmp);
+
+        /* Get the seconds */
+        tmp = PyObject_GetAttrString(obj, "seconds");
+        if (tmp == NULL) {
+            return -1;
+        }
+        seconds = PyInt_AsLong(tmp);
+        if (seconds == -1 && PyErr_Occurred()) {
+            Py_DECREF(tmp);
+            return -1;
+        }
+        Py_DECREF(tmp);
+
+        /* Get the microseconds */
+        tmp = PyObject_GetAttrString(obj, "microseconds");
+        if (tmp == NULL) {
+            return -1;
+        }
+        useconds = PyInt_AsLong(tmp);
+        if (useconds == -1 && PyErr_Occurred()) {
+            Py_DECREF(tmp);
+            return -1;
+        }
+        Py_DECREF(tmp);
+
+        /*
+         * Convert to a microseconds timedelta, then cast to the
+         * desired units.
+         */
+        td = days*(24*60*60*1000000LL) + seconds*1000000LL + useconds;
+        us_meta.base = NPY_FR_us;
+        us_meta.num = 1;
+        us_meta.events = 1;
+        
+        return cast_timedelta_to_timedelta(&us_meta, meta, td, out);
+    }
 
     PyErr_SetString(PyExc_ValueError,
             "Could not convert object to NumPy timedelta");
@@ -3243,6 +3313,127 @@ convert_datetime_to_pyobject(npy_datetime dt, PyArray_DatetimeMetaData *meta)
 }
 
 /*
+ * Converts a timedelta into a PyObject *.
+ *
+ * Not-a-time is returned as the string "NaT".
+ * For microseconds or coarser, returns a datetime.timedelta.
+ * For units finer than microseconds, returns an integer.
+ */
+NPY_NO_EXPORT PyObject *
+convert_timedelta_to_pyobject(npy_timedelta td, PyArray_DatetimeMetaData *meta)
+{
+    PyObject *ret = NULL, *tup = NULL;
+    npy_timedelta value;
+    int event = 0;
+    int days = 0, seconds = 0, useconds = 0;
+
+    /* Handle not-a-time */
+    if (td == NPY_DATETIME_NAT) {
+        return PyUString_FromString("NaT");
+    }
+
+    /*
+     * If the type's precision is greater than microseconds or is
+     * Y/M/B (nonlinear units), return an int
+     */
+    if (meta->base > NPY_FR_us ||
+                    meta->base == NPY_FR_Y ||
+                    meta->base == NPY_FR_M ||
+                    meta->base == NPY_FR_B) {
+        /* Skip use of a tuple for the events, just return the raw int */
+        return PyLong_FromLongLong(td);
+    }
+
+    value = td;
+
+    /* If there are events, extract the event */
+    if (meta->events > 1) {
+        event = (int)(value % meta->events);
+        value = value / meta->events;
+        if (event < 0) {
+            --value;
+            event += meta->events;
+        }
+    }
+
+    /* Apply the unit multiplier (TODO: overflow treatment...) */
+    value *= meta->num;
+
+    /* Convert to days/seconds/useconds */
+    switch (meta->base) {
+        case NPY_FR_W:
+            value *= 7;
+            break;
+        case NPY_FR_D:
+            break;
+        case NPY_FR_h:
+            seconds = (int)((value % 24) * (60*60));
+            value = value / 24;
+            break;
+        case NPY_FR_m:
+            seconds = (int)(value % (24*60)) * 60;
+            value = value / (24*60);
+            break;
+        case NPY_FR_s:
+            seconds = (int)(value % (24*60*60));
+            value = value / (24*60*60);
+            break;
+        case NPY_FR_ms:
+            useconds = (int)(value % 1000) * 1000;
+            value = value / 1000;
+            seconds = (int)(value % (24*60*60));
+            value = value / (24*60*60);
+            break;
+        case NPY_FR_us:
+            useconds = (int)(value % (1000*1000));
+            value = value / (1000*1000);
+            seconds = (int)(value % (24*60*60));
+            value = value / (24*60*60);
+            break;
+        default:
+            break;
+    }
+    /*
+     * 'value' represents days, and seconds/useconds are filled.
+     *
+     * If it would overflow the datetime.timedelta days, return a raw int
+     */
+    if (value < -999999999 || value > 999999999) {
+        return PyLong_FromLongLong(td);
+    }
+    else {
+        days = (int)value;
+        ret = PyDelta_FromDSU(days, seconds, useconds);
+        if (ret == NULL) {
+            return NULL;
+        }
+    }
+
+    /* If there is one event, just return the datetime */
+    if (meta->events == 1) {
+        return ret;
+    }
+    /* Otherwise return a tuple with the event in the second position */
+    else {
+        tup = PyTuple_New(2);
+        if (tup == NULL) {
+            Py_DECREF(ret);
+            return NULL;
+        }
+        PyTuple_SET_ITEM(tup, 0, ret);
+
+        ret = PyInt_FromLong(event);
+        if (ret == NULL) {
+            Py_DECREF(tup);
+            return NULL;
+        }
+        PyTuple_SET_ITEM(tup, 1, ret);
+
+        return tup;
+    }
+}
+
+/*
  * Returns true if the datetime metadata matches
  */
 NPY_NO_EXPORT npy_bool
diff --git a/numpy/core/tests/test_datetime.py b/numpy/core/tests/test_datetime.py
index fe130fb05..1059aeaa1 100644
--- a/numpy/core/tests/test_datetime.py
+++ b/numpy/core/tests/test_datetime.py
@@ -99,6 +99,17 @@ class TestDateTime(TestCase):
         assert_equal(np.datetime64(b, 's'), a)
         assert_equal(np.datetime64(b, 's').dtype, np.dtype('M8[s]'))
 
+        # Construction from datetime.date
+        assert_equal(np.datetime64('1945-03-25'),
+                     np.datetime64(datetime.date(1945,3,25)))
+        assert_equal(np.datetime64('2045-03-25', 'D'),
+                     np.datetime64(datetime.date(2045,3,25), 'D'))
+        # Construction from datetime.datetime
+        assert_equal(np.datetime64('1980-01-25T14:36:22.5Z'),
+                     np.datetime64(datetime.datetime(1980,1,25,
+                                                14,36,22,500000)))
+        
+
     def test_timedelta_scalar_construction(self):
         # Construct with different units
         assert_equal(np.timedelta64(7, 'D'),
@@ -126,6 +137,26 @@ class TestDateTime(TestCase):
         assert_equal(np.timedelta64(b, 's'), a)
         assert_equal(np.timedelta64(b, 's').dtype, np.dtype('m8[s]'))
 
+        # Construction from datetime.timedelta
+        assert_equal(np.timedelta64(5, 'D'),
+                     np.timedelta64(datetime.timedelta(days=5)))
+        assert_equal(np.timedelta64(102347621, 's'),
+                     np.timedelta64(datetime.timedelta(seconds=102347621)))
+        assert_equal(np.timedelta64(-10234760000, 'us'),
+                     np.timedelta64(datetime.timedelta(
+                                            microseconds=-10234760000)))
+        assert_equal(np.timedelta64(10234760000, 'us'),
+                     np.timedelta64(datetime.timedelta(
+                                            microseconds=10234760000)))
+        assert_equal(np.timedelta64(1023476, 'ms'),
+                     np.timedelta64(datetime.timedelta(milliseconds=1023476)))
+        assert_equal(np.timedelta64(10, 'm'),
+                     np.timedelta64(datetime.timedelta(minutes=10)))
+        assert_equal(np.timedelta64(281, 'h'),
+                     np.timedelta64(datetime.timedelta(hours=281)))
+        assert_equal(np.timedelta64(28, 'W'),
+                     np.timedelta64(datetime.timedelta(weeks=28)))
+
     def test_datetime_nat_casting(self):
         a = np.array('NaT', dtype='M8[D]')
         b = np.datetime64('NaT', '[D]')