ENH: support timedelta64[ns] as a serialization type in HDFStore for query and append (GH3577)

Author: jreback

doc/source/io.rst

@@ -2009,6 +2009,22 @@ space. These are in terms of the total number of rows in a table.
                        Term('minor_axis', '=', ['A','B']) ],
                 start=0, stop=10)
 
+**Using timedelta64[ns]**
+
+.. versionadded:: 0.13
+
+Beginning in 0.13.0, you can store and query using the ``timedelta64[ns]`` type. Terms can be
+specified in the format: ``<float>(<unit>)``, where float may be signed (and fractional), and unit can be
+``D,s,ms,us,ns`` for the timedelta. Here's an example:
+
+.. ipython:: python
+
+   from datetime import timedelta
+   dftd = DataFrame(dict(A = Timestamp('20130101'), B = [ Timestamp('20130101') + timedelta(days=i,seconds=10) for i in range(10) ]))
+   dftd['C'] = dftd['A']-dftd['B']
+   dftd
+   store.append('dftd',dftd,data_columns=True)
+   store.select('dftd',Term("C","<","-3.5D"))
 
 Indexing
 ~~~~~~~~

doc/source/release.rst

@@ -156,6 +156,7 @@ API Changes
     - a column multi-index will be recreated properly (:issue:`4710`); raise on trying to use a multi-index
       with data_columns on the same axis
     - ``select_as_coordinates`` will now return an ``Int64Index`` of the resultant selection set
+    - support ``timedelta64[ns]`` as a serialization type (:issue:`3577`)
   - ``JSON``
 
     - added ``date_unit`` parameter to specify resolution of timestamps. Options

doc/source/v0.13.0.txt

@@ -80,7 +80,7 @@ API changes
       See :ref:`here<io.hdf5-selecting_coordinates>` for an example.
     - allow a passed locations array or mask as a ``where`` condition (:issue:`4467`).
       See :ref:`here<io.hdf5-where_mask>` for an example.
-
+    - support ``timedelta64[ns]`` as a serialization type (:issue:`3577`)
     - the ``format`` keyword now replaces the ``table`` keyword; allowed values are ``fixed(f)`` or ``table(t)``
       the same defaults as prior < 0.13.0 remain, e.g. ``put`` implies 'fixed` or 'f' (Fixed) format
       and ``append`` imples 'table' or 't' (Table) format

pandas/core/common.py

@@ -19,7 +19,6 @@
 from pandas.core.config import get_option
 from pandas.core import array as pa
 
-
 # XXX: HACK for NumPy 1.5.1 to suppress warnings
 try:
     np.seterr(all='ignore')
@@ -704,13 +703,29 @@ def diff(arr, n, axis=0):
 
     return out_arr
 
+timedelta_search = re.compile(
+    "^(?P<value>-?\d*\.?\d*)(?P<unit>D|s|ms|us|ns)?$")
 
-def _coerce_scalar_to_timedelta_type(r):
+def _coerce_scalar_to_timedelta_type(r, unit='ns'):
     # kludgy here until we have a timedelta scalar
     # handle the numpy < 1.7 case
 
+    if isinstance(r, compat.string_types):
+        m = timedelta_search.search(r)
+        if m:
+            r = float(m.groupdict()['value'])
+            u = m.groupdict().get('unit')
+            if u is not None:
+                unit = u
+        else:
+            raise ValueError("cannot convert timedelta scalar value!")
+
+        r = tslib.cast_from_unit(unit, r)
+        r = timedelta(microseconds=int(r)/1000)
+
     if is_integer(r):
-        r = timedelta(microseconds=r/1000)
+        r = tslib.cast_from_unit(unit, r)
+        r = timedelta(microseconds=int(r)/1000)
 
     if _np_version_under1p7:
         if not isinstance(r, timedelta):

pandas/io/pytables.py

@@ -25,7 +25,7 @@
 from pandas.core.common import adjoin, is_list_like, pprint_thing
 from pandas.core.algorithms import match, unique
 from pandas.core.categorical import Categorical
-from pandas.core.common import _asarray_tuplesafe
+from pandas.core.common import _asarray_tuplesafe, _np_version_under1p7
 from pandas.core.internals import BlockManager, make_block
 from pandas.core.reshape import block2d_to_blocknd, factor_indexer
 from pandas.core.index import _ensure_index
@@ -1527,6 +1527,8 @@ def set_kind(self):
                 self.kind = 'integer'
             elif dtype.startswith(u('date')):
                 self.kind = 'datetime'
+            elif dtype.startswith(u('timedelta')):
+                self.kind = 'timedelta'
             elif dtype.startswith(u('bool')):
                 self.kind = 'bool'
             else:
@@ -1547,6 +1549,11 @@ def set_atom(self, block, existing_col, min_itemsize, nan_rep, info, encoding=No
 
         if inferred_type == 'datetime64':
             self.set_atom_datetime64(block)
+        elif dtype == 'timedelta64[ns]':
+            if _np_version_under1p7:
+                raise TypeError(
+                    "timdelta64 is not supported under under numpy < 1.7")
+            self.set_atom_timedelta64(block)
         elif inferred_type == 'date':
             raise TypeError(
                 "[date] is not implemented as a table column")
@@ -1667,6 +1674,16 @@ def set_atom_datetime64(self, block, values=None):
             values = block.values.view('i8')
         self.set_data(values, 'datetime64')
 
+    def get_atom_timedelta64(self, block):
+        return _tables().Int64Col(shape=block.shape[0])
+
+    def set_atom_timedelta64(self, block, values=None):
+        self.kind = 'timedelta64'
+        self.typ = self.get_atom_timedelta64(block)
+        if values is None:
+            values = block.values.view('i8')
+        self.set_data(values, 'timedelta64')
+
     @property
     def shape(self):
         return getattr(self.data, 'shape', None)
@@ -1719,6 +1736,8 @@ def convert(self, values, nan_rep, encoding):
                 else:
                     self.data = np.asarray(self.data, dtype='M8[ns]')
 
+            elif dtype == u('timedelta64'):
+                self.data = np.asarray(self.data, dtype='m8[ns]')
             elif dtype == u('date'):
                 self.data = np.array(
                     [date.fromtimestamp(v) for v in self.data], dtype=object)
@@ -1767,6 +1786,9 @@ def get_atom_data(self, block):
     def get_atom_datetime64(self, block):
         return _tables().Int64Col()
 
+    def get_atom_timedelta64(self, block):
+        return _tables().Int64Col()
+
 
 class GenericDataIndexableCol(DataIndexableCol):
 
@@ -2007,6 +2029,11 @@ def read_array(self, key):
 
             if dtype == u('datetime64'):
                 ret = np.array(ret, dtype='M8[ns]')
+            elif dtype == u('timedelta64'):
+                if _np_version_under1p7:
+                    raise TypeError(
+                        "timedelta64 is not supported under under numpy < 1.7")
+                ret = np.array(ret, dtype='m8[ns]')
 
         if transposed:
             return ret.T
@@ -2214,6 +2241,9 @@ def write_array(self, key, value, items=None):
         elif value.dtype.type == np.datetime64:
             self._handle.createArray(self.group, key, value.view('i8'))
             getattr(self.group, key)._v_attrs.value_type = 'datetime64'
+        elif value.dtype.type == np.timedelta64:
+            self._handle.createArray(self.group, key, value.view('i8'))
+            getattr(self.group, key)._v_attrs.value_type = 'timedelta64'
         else:
             if empty_array:
                 self.write_array_empty(key, value)
@@ -4000,7 +4030,9 @@ def eval(self):
         """ set the numexpr expression for this term """
 
         if not self.is_valid:
-            raise ValueError("query term is not valid [%s]" % str(self))
+            raise ValueError("query term is not valid [{0}]\n"
+                             "  all queries terms must include a reference to\n"
+                             "  either an axis (e.g. index or column), or a data_columns\n".format(str(self)))
 
         # convert values if we are in the table
         if self.is_in_table:
@@ -4060,6 +4092,9 @@ def stringify(value):
             if v.tz is not None:
                 v = v.tz_convert('UTC')
             return TermValue(v, v.value, kind)
+        elif kind == u('timedelta64') or kind == u('timedelta'):
+            v = com._coerce_scalar_to_timedelta_type(v,unit='s').item()
+            return TermValue(int(v), v, kind)
         elif (isinstance(v, datetime) or hasattr(v, 'timetuple')
               or kind == u('date')):
             v = time.mktime(v.timetuple())

pandas/io/tests/test_pytables.py

@@ -23,6 +23,7 @@
                                  assert_series_equal)
 from pandas import concat, Timestamp
 from pandas import compat
+from pandas.core import common as com
 
 from numpy.testing.decorators import slow
 
@@ -1732,7 +1733,7 @@ def test_unimplemented_dtypes_table_columns(self):
             # this fails because we have a date in the object block......
             self.assertRaises(TypeError, store.append, 'df_unimplemented', df)
 
-    def test_table_append_with_timezones(self):
+    def test_append_with_timezones(self):
 
         from datetime import timedelta
 
@@ -1798,6 +1799,51 @@ def compare(a,b):
             result = store.select('df')
             assert_frame_equal(result,df)
 
+    def test_append_with_timedelta(self):
+        if com._np_version_under1p7:
+            raise nose.SkipTest("requires numpy >= 1.7")
+
+        # GH 3577
+        # append timedelta
+
+        from datetime import timedelta
+        df = DataFrame(dict(A = Timestamp('20130101'), B = [ Timestamp('20130101') + timedelta(days=i,seconds=10) for i in range(10) ]))
+        df['C'] = df['A']-df['B']
+        df.ix[3:5,'C'] = np.nan
+
+        with ensure_clean(self.path) as store:
+
+            # table
+            _maybe_remove(store, 'df')
+            store.append('df',df,data_columns=True)
+            result = store.select('df')
+            assert_frame_equal(result,df)
+
+            result = store.select('df',Term("C<100000"))
+            assert_frame_equal(result,df)
+
+            result = store.select('df',Term("C","<",-3*86400))
+            assert_frame_equal(result,df.iloc[3:])
+
+            result = store.select('df',Term("C","<",'-3D'))
+            assert_frame_equal(result,df.iloc[3:])
+
+            # a bit hacky here as we don't really deal with the NaT properly
+
+            result = store.select('df',Term("C","<",'-500000s'))
+            result = result.dropna(subset=['C'])
+            assert_frame_equal(result,df.iloc[6:])
+
+            result = store.select('df',Term("C","<",'-3.5D'))
+            result = result.iloc[1:]
+            assert_frame_equal(result,df.iloc[4:])
+
+            # fixed
+            _maybe_remove(store, 'df2')
+            store.put('df2',df)
+            result = store.select('df2')
+            assert_frame_equal(result,df)
+
     def test_remove(self):
 
         with ensure_clean(self.path) as store:

pandas/tslib.pyx

@@ -1276,7 +1276,7 @@ cdef inline _get_datetime64_nanos(object val):
     else:
         return ival
 
-cdef inline int64_t cast_from_unit(object unit, object ts) except -1:
+cpdef inline int64_t cast_from_unit(object unit, object ts) except -1:
     """ return a casting of the unit represented to nanoseconds
         round the fractional part of a float to our precision, p """
     if unit == 'D':