TST: add pandas/tseries/tests/test_timedeltas.py

API: add full timedelta parsing and conversion to np.timedelta64[ns]

Author: jreback

doc/source/release.rst

@@ -191,6 +191,7 @@ API Changes
   - provide automatic dtype conversions on _reduce operations (:issue:`3371`)
   - exclude non-numerics if mixed types with datelike in _reduce operations (:issue:`3371`)
   - default for ``tupleize_cols`` is now ``False`` for both ``to_csv`` and ``read_csv``. Fair warning in 0.12 (:issue:`3604`)
+  - moved timedeltas support to pandas.tseries.timedeltas.py; add timedeltas string parsing
 
 Internal Refactoring
 ~~~~~~~~~~~~~~~~~~~~

pandas/tseries/tests/test_timedeltas.py

@@ -0,0 +1,128 @@
+# pylint: disable-msg=E1101,W0612
+
+from datetime import datetime, timedelta
+import nose
+import unittest
+
+import numpy as np
+import pandas as pd
+
+from pandas import (Index, Series, DataFrame, isnull, notnull,
+                    bdate_range, date_range, _np_version_under1p7)
+import pandas.core.common as com
+from pandas.compat import StringIO, lrange, range, zip, u, OrderedDict, long
+from pandas import compat
+from pandas.tseries.timedeltas import _coerce_scalar_to_timedelta_type as ct
+from pandas.util.testing import (assert_series_equal,
+                                 assert_frame_equal,
+                                 assert_almost_equal,
+                                 ensure_clean)
+import pandas.util.testing as tm
+
+def _skip_if_numpy_not_friendly():
+    # not friendly for < 1.7
+    if _np_version_under1p7:
+        raise nose.SkipTest("numpy < 1.7")
+
+class TestTimedeltas(unittest.TestCase):
+    _multiprocess_can_split_ = True
+
+    def setUp(self):
+        pass
+
+    def test_numeric_conversions(self):
+        _skip_if_numpy_not_friendly()
+
+        # ns not converted properly
+        self.assert_(ct(0) == np.timedelta64(0,'ns'))
+        self.assert_(ct(10) == np.timedelta64(0,'ns'))
+        self.assert_(ct(10,unit='ns') == np.timedelta64(0,'ns').astype('m8[ns]'))
+
+        self.assert_(ct(10,unit='us') == np.timedelta64(10,'us').astype('m8[ns]'))
+        self.assert_(ct(10,unit='ms') == np.timedelta64(10,'ms').astype('m8[ns]'))
+        self.assert_(ct(10,unit='s') == np.timedelta64(10,'s').astype('m8[ns]'))
+        self.assert_(ct(10,unit='d') == np.timedelta64(10,'D').astype('m8[ns]'))
+
+    def test_timedelta_conversions(self):
+        _skip_if_numpy_not_friendly()
+
+        self.assert_(ct(timedelta(seconds=1)) == np.timedelta64(1,'s').astype('m8[ns]'))
+        self.assert_(ct(timedelta(microseconds=1)) == np.timedelta64(1,'us').astype('m8[ns]'))
+        self.assert_(ct(timedelta(days=1)) == np.timedelta64(1,'D').astype('m8[ns]'))
+
+    def test_short_format_converters(self):
+        _skip_if_numpy_not_friendly()
+
+        def conv(v):
+            return v.astype('m8[ns]')
+
+        # ns not converted properly
+        self.assert_(ct('10') == np.timedelta64(0,'ns'))
+        self.assert_(ct('10ns') == np.timedelta64(0,'ns'))
+        self.assert_(ct('100') == np.timedelta64(0,'ns'))
+        self.assert_(ct('100ns') == np.timedelta64(0,'ns'))
+
+        self.assert_(ct('1000') == np.timedelta64(1000,'ns'))
+        self.assert_(ct('1000ns') == np.timedelta64(1000,'ns'))
+        self.assert_(ct('1000NS') == np.timedelta64(1000,'ns'))
+
+        self.assert_(ct('10us') == np.timedelta64(10000,'ns'))
+        self.assert_(ct('100us') == np.timedelta64(100000,'ns'))
+        self.assert_(ct('1000us') == np.timedelta64(1000000,'ns'))
+        self.assert_(ct('1000Us') == np.timedelta64(1000000,'ns'))
+        self.assert_(ct('1000uS') == np.timedelta64(1000000,'ns'))
+
+        self.assert_(ct('1ms') == np.timedelta64(1000000,'ns'))
+        self.assert_(ct('10ms') == np.timedelta64(10000000,'ns'))
+        self.assert_(ct('100ms') == np.timedelta64(100000000,'ns'))
+        self.assert_(ct('1000ms') == np.timedelta64(1000000000,'ns'))
+
+        self.assert_(ct('-1s') == -np.timedelta64(1000000000,'ns'))
+        self.assert_(ct('1s') == np.timedelta64(1000000000,'ns'))
+        self.assert_(ct('10s') == np.timedelta64(10000000000,'ns'))
+        self.assert_(ct('100s') == np.timedelta64(100000000000,'ns'))
+        self.assert_(ct('1000s') == np.timedelta64(1000000000000,'ns'))
+
+        self.assert_(ct('1d') == conv(np.timedelta64(1,'D')))
+        self.assert_(ct('-1d') == -conv(np.timedelta64(1,'D')))
+        self.assert_(ct('1D') == conv(np.timedelta64(1,'D')))
+        self.assert_(ct('10D') == conv(np.timedelta64(10,'D')))
+        self.assert_(ct('100D') == conv(np.timedelta64(100,'D')))
+        self.assert_(ct('1000D') == conv(np.timedelta64(1000,'D')))
+        self.assert_(ct('10000D') == conv(np.timedelta64(10000,'D')))
+
+        # space
+        self.assert_(ct(' 10000D ') == conv(np.timedelta64(10000,'D')))
+        self.assert_(ct(' - 10000D ') == -conv(np.timedelta64(10000,'D')))
+
+        # invalid
+        self.assertRaises(ValueError, ct, '1foo')
+        self.assertRaises(ValueError, ct, 'foo')
+
+    def test_full_format_converters(self):
+        _skip_if_numpy_not_friendly()
+
+        def conv(v):
+            return v.astype('m8[ns]')
+        d1 = np.timedelta64(1,'D')
+
+        self.assert_(ct('1days') == conv(d1))
+        self.assert_(ct('1days,') == conv(d1))
+        self.assert_(ct('- 1days,') == -conv(d1))
+
+        self.assert_(ct('00:00:01') == conv(np.timedelta64(1,'s')))
+        self.assert_(ct('06:00:01') == conv(np.timedelta64(6*3600+1,'s')))
+        self.assert_(ct('06:00:01.0') == conv(np.timedelta64(6*3600+1,'s')))
+        self.assert_(ct('06:00:01.01') == conv(np.timedelta64(1000*(6*3600+1)+10,'ms')))
+
+        self.assert_(ct('- 1days, 00:00:01') == -conv(d1+np.timedelta64(1,'s')))
+        self.assert_(ct('1days, 06:00:01') == conv(d1+np.timedelta64(6*3600+1,'s')))
+        self.assert_(ct('1days, 06:00:01.01') == conv(d1+np.timedelta64(1000*(6*3600+1)+10,'ms')))
+
+        # invalid
+        self.assertRaises(ValueError, ct, '- 1days, 00')
+
+
+if __name__ == '__main__':
+    nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
+                   exit=False)

pandas/tseries/timedeltas.py

@@ -8,31 +8,29 @@
 import numpy as np
 import pandas.tslib as tslib
 from pandas import compat, _np_version_under1p7
-from pandas.core.common import ABCSeries, is_integer, _values_from_object
+from pandas.core.common import (ABCSeries, is_integer,
+                                _values_from_object, is_list_like)
 
-timedelta_search = re.compile(
-    "^(?P<value>-?\d*\.?\d*)(?P<unit>D|s|ms|us|ns)?$")
+repr_timedelta = tslib.repr_timedelta64
+repr_timedelta64 = tslib.repr_timedelta64
+
+_short_search = re.compile(
+    "^\s*(?P<neg>-?)\s*(?P<value>\d*\.?\d*)\s*(?P<unit>d|s|ms|us|ns)?\s*$",re.IGNORECASE)
+_full_search = re.compile(
+    "^\s*(?P<neg>-?)\s*(?P<days>\d+)?\s*(days|d)?,?\s*(?P<time>\d{2}:\d{2}:\d{2})?(?P<frac>\.\d+)?\s*$",re.IGNORECASE)
 
 def _coerce_scalar_to_timedelta_type(r, unit='ns'):
     # kludgy here until we have a timedelta scalar
     # handle the numpy < 1.7 case
 
     if isinstance(r, compat.string_types):
-        m = timedelta_search.search(r)
-        if m:
-            r = float(m.groupdict()['value'])
-            u = m.groupdict().get('unit')
-            if u is not None:
-                unit = u
-        else:
-            raise ValueError("cannot convert timedelta scalar value!")
-
-        r = tslib.cast_from_unit(unit, r)
-        r = timedelta(microseconds=int(r)/1000)
+        converter = _get_string_converter(r, unit=unit)
+        r = converter()
+        r = timedelta(microseconds=r/1000.0)
 
     if is_integer(r):
-        r = tslib.cast_from_unit(unit, r)
-        r = timedelta(microseconds=int(r)/1000)
+        r = tslib.cast_from_unit(r, unit)
+        r = timedelta(microseconds=r/1000.0)
 
     if _np_version_under1p7:
         if not isinstance(r, timedelta):
@@ -49,6 +47,54 @@ def _coerce_scalar_to_timedelta_type(r, unit='ns'):
         raise AssertionError("Invalid type for timedelta scalar: %s" % type(r))
     return r.astype('timedelta64[ns]')
 
+def _get_string_converter(r, unit='ns'):
+    """ return a string converter for r to process the timedelta format """
+
+    m = _short_search.search(r)
+    if m:
+        def convert(r=None, unit=unit, m=m):
+            if r is not None:
+                m = _short_search.search(r)
+
+            gd = m.groupdict()
+
+            r = float(gd['value'])
+            u = gd.get('unit')
+            if u is not None:
+                unit = u.lower()
+            if gd['neg']:
+                r *= -1
+            return tslib.cast_from_unit(r, unit)
+        return convert
+
+    m = _full_search.search(r)
+    if m:
+        def convert(r=None, unit=None, m=m):
+            if r is not None:
+                m = _full_search.search(r)
+
+            gd = m.groupdict()
+
+            # convert to seconds
+            value = float(gd['days'] or 0) * 86400
+
+            time = gd['time']
+            if time:
+                (hh,mm,ss) = time.split(':')
+                value += float(hh)*3600 + float(mm)*60 + float(ss)
+
+            frac = gd['frac']
+            if frac:
+                value += float(frac)
+
+            if gd['neg']:
+                value *= -1
+            return tslib.cast_from_unit(value, 's')
+        return convert
+
+    # no converter
+    raise ValueError("cannot create timedelta string converter")
+
 def _possibly_cast_to_timedelta(value, coerce=True):
     """ try to cast to timedelta64, if already a timedeltalike, then make
         sure that we are [ns] (as numpy 1.6.2 is very buggy in this regards,

pandas/tslib.pyx

@@ -665,14 +665,14 @@ cdef convert_to_tsobject(object ts, object tz, object unit):
         if ts == NPY_NAT:
             obj.value = NPY_NAT
         else:
-            ts = ts * cast_from_unit(unit,None)
+            ts = ts * cast_from_unit(None,unit)
             obj.value = ts
             pandas_datetime_to_datetimestruct(ts, PANDAS_FR_ns, &obj.dts)
     elif util.is_float_object(ts):
         if ts != ts or ts == NPY_NAT:
             obj.value = NPY_NAT
         else:
-            ts = cast_from_unit(unit,ts)
+            ts = cast_from_unit(ts,unit)
             obj.value = ts
             pandas_datetime_to_datetimestruct(ts, PANDAS_FR_ns, &obj.dts)
     elif util.is_string_object(ts):
@@ -852,7 +852,7 @@ def array_to_datetime(ndarray[object] values, raise_=False, dayfirst=False,
         pandas_datetimestruct dts
         bint utc_convert = bool(utc)
         _TSObject _ts
-        int64_t m = cast_from_unit(unit,None)
+        int64_t m = cast_from_unit(None,unit)
 
     try:
         result = np.empty(n, dtype='M8[ns]')
@@ -892,7 +892,7 @@ def array_to_datetime(ndarray[object] values, raise_=False, dayfirst=False,
                 if val != val or val == iNaT:
                     iresult[i] = iNaT
                 else:
-                    iresult[i] = cast_from_unit(unit,val)
+                    iresult[i] = cast_from_unit(val,unit)
             else:
                 try:
                     if len(val) == 0:
@@ -1276,10 +1276,10 @@ cdef inline _get_datetime64_nanos(object val):
     else:
         return ival
 
-cpdef inline int64_t cast_from_unit(object unit, object ts) except -1:
+cpdef inline int64_t cast_from_unit(object ts, object unit) except -1:
     """ return a casting of the unit represented to nanoseconds
         round the fractional part of a float to our precision, p """
-    if unit == 'D':
+    if unit == 'D' or unit == 'd':
         m = 1000000000L * 86400
         p = 6
     elif unit == 's':